From 5c99f0fa58caf45f4d457894413aa11c03afdb7d Mon Sep 17 00:00:00 2001 From: Nick Date: Fri, 13 Sep 2019 07:48:07 -0400 Subject: Revert "Add src/cuda-sim formatting" This reverts commit 0c023e41809dba8897c37af6bb03e5c3aa9ebc5e. --- src/cuda-sim/cuda-math.h | 455 +- src/cuda-sim/cuda-sim.cc | 4899 ++++++++------- src/cuda-sim/cuda-sim.h | 287 +- src/cuda-sim/cuda_device_printf.cc | 165 +- src/cuda-sim/cuda_device_printf.h | 10 +- src/cuda-sim/cuda_device_runtime.cc | 570 +- src/cuda-sim/cuda_device_runtime.h | 91 +- src/cuda-sim/half.h | 6539 +++++++++----------- src/cuda-sim/instructions.cc | 10876 ++++++++++++++++------------------ src/cuda-sim/memory.cc | 336 +- src/cuda-sim/memory.h | 158 +- src/cuda-sim/opcodes.h | 77 +- src/cuda-sim/ptx-stats.cc | 376 +- src/cuda-sim/ptx-stats.h | 54 +- src/cuda-sim/ptx_ir.cc | 2665 ++++----- src/cuda-sim/ptx_ir.h | 2947 ++++----- src/cuda-sim/ptx_loader.cc | 951 ++- src/cuda-sim/ptx_loader.h | 38 +- src/cuda-sim/ptx_parser.cc | 1785 +++--- src/cuda-sim/ptx_parser.h | 302 +- src/cuda-sim/ptx_sim.cc | 1061 ++-- src/cuda-sim/ptx_sim.h | 912 +-- 22 files changed, 16746 insertions(+), 18808 deletions(-) (limited to 'src/cuda-sim') diff --git a/src/cuda-sim/cuda-math.h b/src/cuda-sim/cuda-math.h index 97ce130..9a5468c 100644 --- a/src/cuda-sim/cuda-math.h +++ b/src/cuda-sim/cuda-math.h @@ -1,6 +1,6 @@ // This file created from vector_types.h distributed with CUDA 1.1 // (see original copyright notice below) -// +// // Changes Copyright (c) 2009-2011, Tor M. Aamodt, Wilson W.L. Fung // The University of British Columbia // All rights reserved. @@ -10,16 +10,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -30,41 +28,43 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + /* * Copyright 1993-2007 NVIDIA Corporation. All rights reserved. * - * NOTICE TO USER: + * NOTICE TO USER: * - * This source code is subject to NVIDIA ownership rights under U.S. and - * international Copyright laws. Users and possessors of this source code - * are hereby granted a nonexclusive, royalty-free license to use this code + * This source code is subject to NVIDIA ownership rights under U.S. and + * international Copyright laws. Users and possessors of this source code + * are hereby granted a nonexclusive, royalty-free license to use this code * in individual and commercial software. * - * NVIDIA MAKES NO REPRESENTATION ABOUT THE SUITABILITY OF THIS SOURCE - * CODE FOR ANY PURPOSE. IT IS PROVIDED "AS IS" WITHOUT EXPRESS OR - * IMPLIED WARRANTY OF ANY KIND. NVIDIA DISCLAIMS ALL WARRANTIES WITH - * REGARD TO THIS SOURCE CODE, INCLUDING ALL IMPLIED WARRANTIES OF + * NVIDIA MAKES NO REPRESENTATION ABOUT THE SUITABILITY OF THIS SOURCE + * CODE FOR ANY PURPOSE. IT IS PROVIDED "AS IS" WITHOUT EXPRESS OR + * IMPLIED WARRANTY OF ANY KIND. NVIDIA DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOURCE CODE, INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY, NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. - * IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL, - * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS - * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE - * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE - * OR PERFORMANCE OF THIS SOURCE CODE. + * IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL, + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE + * OR PERFORMANCE OF THIS SOURCE CODE. * - * U.S. Government End Users. This source code is a "commercial item" as - * that term is defined at 48 C.F.R. 2.101 (OCT 1995), consisting of - * "commercial computer software" and "commercial computer software - * documentation" as such terms are used in 48 C.F.R. 12.212 (SEPT 1995) - * and is provided to the U.S. Government only as a commercial end item. - * Consistent with 48 C.F.R.12.212 and 48 C.F.R. 227.7202-1 through - * 227.7202-4 (JUNE 1995), all U.S. Government End Users acquire the - * source code with only those rights set forth herein. + * U.S. Government End Users. This source code is a "commercial item" as + * that term is defined at 48 C.F.R. 2.101 (OCT 1995), consisting of + * "commercial computer software" and "commercial computer software + * documentation" as such terms are used in 48 C.F.R. 12.212 (SEPT 1995) + * and is provided to the U.S. Government only as a commercial end item. + * Consistent with 48 C.F.R.12.212 and 48 C.F.R. 227.7202-1 through + * 227.7202-4 (JUNE 1995), all U.S. Government End Users acquire the + * source code with only those rights set forth herein. * - * Any use of this source code in individual and commercial software must + * Any use of this source code in individual and commercial software must * include, in the user documentation and internal comments to the code, * the above Disclaimer and U.S. Government End Users Notice. */ + #ifndef CUDA_MATH #define CUDA_MATH @@ -74,31 +74,32 @@ #undef max #undef min namespace cuda_math { -#define __attribute__(a) // to remove warnings inside math_functions.h +#define __attribute__(a) // to remove warnings inside math_functions.h #undef INT_MAX #if CUDART_VERSION < 3000 // DEVICE_BUILTIN -struct int4 { - int x, y, z, w; -}; -struct uint4 { - unsigned int x, y, z, w; -}; -struct float4 { - float x, y, z, w; -}; -struct float2 { - float x, y; -}; + struct int4 { + int x, y, z, w; + }; + struct uint4 { + unsigned int x, y, z, w; + }; + struct float4 { + float x, y, z, w; + }; + struct float2 { + float x, y; + }; + // DEVICE_BUILTIN -typedef struct int4 int4; -typedef struct uint4 uint4; -typedef struct float4 float4; -typedef struct float2 float2; + typedef struct int4 int4; + typedef struct uint4 uint4; + typedef struct float4 float4; + typedef struct float2 float2; -extern float rsqrtf(float); // CUDA 2.3 beta +extern float rsqrtf(float); // CUDA 2.3 beta #define CUDA_FLOAT_MATH_FUNCTIONS #include @@ -107,36 +108,38 @@ extern float rsqrtf(float); // CUDA 2.3 beta #undef __CUDA_INTERNAL_COMPILATION__ #undef __attribute__ -// float to integer conversion -int float2int(float a, enum cudaRoundMode mode) { - return __internal_float2uint(a, mode); +// float to integer conversion +int float2int(float a, enum cudaRoundMode mode) +{ + return __internal_float2uint(a, mode); } -// float to unsigned integer conversion -unsigned int float2uint(float a, enum cudaRoundMode mode) { - return __internal_float2uint(a, mode); +// float to unsigned integer conversion +unsigned int float2uint(float a, enum cudaRoundMode mode) +{ + return __internal_float2uint(a, mode); } float __ll2float_rz(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_ru(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rd(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } #else @@ -144,211 +147,205 @@ float __ll2float_rd(long long int a) { #define CUDA_FLOAT_MATH_FUNCTIONS #define __CUDACC__ -// implementing int to float intrinsics with different rounding modes +// implementing int to float intrinsics with different rounding modes #include #include + // 32-bit integer to float float __int2float_rn(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_rz(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_ru(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_rd(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } // 32-bit unsigned integer to float float __uint2float_rn(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_rz(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_ru(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_rd(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } // 64-bit integer to float float __ll2float_rn(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rz(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_ru(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rd(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } -// 64-bit unsigned integer to float +// 64-bit unsigned integer to float float __ull2float_rn(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_rz(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_ru(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_rd(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; -} - -// float to integer conversion -int float2int(float a, enum cudaRoundMode mode) { - int tmp; - switch (mode) { - case cudaRoundZero: - tmp = truncf(a); - break; - case cudaRoundNearest: - tmp = nearbyintf(a); - break; - case cudaRoundMinInf: - tmp = floorf(a); - break; - case cudaRoundPosInf: - tmp = ceilf(a); - break; - default: - abort(); - } - return tmp; -} - -int __internal_float2int(float a, enum cudaRoundMode mode) { - return float2int(a, mode); -} - -// float to unsigned integer conversion -unsigned int float2uint(float a, enum cudaRoundMode mode) { - unsigned int tmp; - switch (mode) { - case cudaRoundZero: - tmp = truncf(a); - break; - case cudaRoundNearest: - tmp = nearbyintf(a); - break; - case cudaRoundMinInf: - tmp = floorf(a); - break; - case cudaRoundPosInf: - tmp = ceilf(a); - break; - default: - abort(); - } - return tmp; -} - -unsigned int __internal_float2uint(float a, enum cudaRoundMode mode) { - return float2uint(a, mode); -} - -// intrinsic for division -float fdividef(float a, float b) { return (a / b); } - -float __internal_accurate_fdividef(float a, float b) { return fdividef(a, b); } + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; +} + +// float to integer conversion +int float2int(float a, enum cudaRoundMode mode) +{ + int tmp; + switch (mode) { + case cudaRoundZero: tmp = truncf(a); break; + case cudaRoundNearest: tmp = nearbyintf(a); break; + case cudaRoundMinInf: tmp = floorf(a); break; + case cudaRoundPosInf: tmp = ceilf(a); break; + default: abort(); + } + return tmp; +} + +int __internal_float2int(float a, enum cudaRoundMode mode) +{ + return float2int(a, mode); +} + +// float to unsigned integer conversion +unsigned int float2uint(float a, enum cudaRoundMode mode) +{ + unsigned int tmp; + switch (mode) { + case cudaRoundZero: tmp = truncf(a); break; + case cudaRoundNearest: tmp = nearbyintf(a); break; + case cudaRoundMinInf: tmp = floorf(a); break; + case cudaRoundPosInf: tmp = ceilf(a); break; + default: abort(); + } + return tmp; +} + +unsigned int __internal_float2uint(float a, enum cudaRoundMode mode) +{ + return float2uint(a, mode); +} + +// intrinsic for division +float fdividef(float a, float b) +{ + return (a / b); +} + +float __internal_accurate_fdividef(float a, float b) +{ + return fdividef(a, b); +} // intrinsic for saturate (clamp values beyond 0 and 1) -float __saturatef(float a) { - float b; - if (std::isnan(a)) - b = 0.0f; - else if (a >= 1.0f) - b = 1.0f; - else if (a <= 0.0f) - b = 0.0f; - else - b = a; - return b; -} - -// intrinsic for power -float __powf(float a, float b) { return powf(a, b); } +float __saturatef(float a) +{ + float b; + if (std::isnan(a)) b = 0.0f; + else if (a >= 1.0f) b = 1.0f; + else if (a <= 0.0f) b = 0.0f; + else b = a; + return b; +} + +// intrinsic for power +float __powf(float a, float b) +{ + return powf(a, b); +} // math functions missing in Mac OSX GCC #ifdef __APPLE__ -int __signbitd(double d) { - unsigned long long int u = *((unsigned long long int*)&d); - return ((u & 0x8000000000000000ULL) != 0); +int __signbitd(double d) +{ + unsigned long long int u = *((unsigned long long int*)&d); + return ((u & 0x8000000000000000ULL) != 0); } -#endif +#endif #undef __CUDACC__ #define __CUDA_INTERNAL_COMPILATION__ @@ -357,11 +354,15 @@ int __signbitd(double d) { #undef __attribute__ #endif + } // math functions missing in Mac OSX GCC #ifdef __APPLE__ -int isnanf(float a) { return (std::isnan(a)); } -#endif +int isnanf(float a) +{ + return (std::isnan(a)); +} +#endif #endif diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index ac8af05..7a130ea 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1,5 +1,5 @@ // Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda, Wilson W.L. Fung, -// George L. Yuan, Jimmy Kwa +// George L. Yuan, Jimmy Kwa // The University of British Columbia // All rights reserved. // @@ -8,16 +8,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -33,318 +31,260 @@ #include "instructions.h" #include "ptx_ir.h" class ptx_recognizer; -typedef void *yyscan_t; +typedef void * yyscan_t; +#include "ptx.tab.h" +#include "ptx_sim.h" #include -#include -#include #include -#include "../../libcuda/gpgpu_context.h" -#include "../abstract_hardware_model.h" -#include "../gpgpu-sim/gpu-sim.h" -#include "../gpgpusim_entrypoint.h" +#include "opcodes.h" #include "../statwrapper.h" -#include "../stream_manager.h" -#include "cuda_device_runtime.h" -#include "decuda_pred_table/decuda_pred_table.h" +#include +#include +#include "../abstract_hardware_model.h" #include "memory.h" -#include "opcodes.h" #include "ptx-stats.h" -#include "ptx.tab.h" #include "ptx_loader.h" #include "ptx_parser.h" +#include "../gpgpu-sim/gpu-sim.h" #include "ptx_sim.h" -#include "ptx_sim.h" +#include "../gpgpusim_entrypoint.h" +#include "decuda_pred_table/decuda_pred_table.h" +#include "../stream_manager.h" +#include "cuda_device_runtime.h" +#include "../../libcuda/gpgpu_context.h" int g_debug_execution = 0; // Output debug information to file options -void cuda_sim::ptx_opcocde_latency_options(option_parser_t opp) { - option_parser_register(opp, "-ptx_opcode_latency_int", OPT_CSTR, - &opcode_latency_int, - "Opcode latencies for integers " - "Default 1,1,19,25,145", - "1,1,19,25,145"); - option_parser_register(opp, "-ptx_opcode_latency_fp", OPT_CSTR, - &opcode_latency_fp, - "Opcode latencies for single precision floating " - "points " - "Default 1,1,1,1,30", - "1,1,1,1,30"); - option_parser_register(opp, "-ptx_opcode_latency_dp", OPT_CSTR, - &opcode_latency_dp, - "Opcode latencies for double precision floating " - "points " - "Default 8,8,8,8,335", - "8,8,8,8,335"); - option_parser_register(opp, "-ptx_opcode_latency_sfu", OPT_CSTR, - &opcode_latency_sfu, - "Opcode latencies for SFU instructions" - "Default 8", - "8"); - option_parser_register(opp, "-ptx_opcode_latency_tesnor", OPT_CSTR, - &opcode_latency_tensor, - "Opcode latencies for Tensor instructions" - "Default 64", - "64"); - option_parser_register( - opp, "-ptx_opcode_initiation_int", OPT_CSTR, &opcode_initiation_int, - "Opcode initiation intervals for integers " - "Default 1,1,4,4,32", - "1,1,4,4,32"); - option_parser_register(opp, "-ptx_opcode_initiation_fp", OPT_CSTR, - &opcode_initiation_fp, - "Opcode initiation intervals for single precision " - "floating points " - "Default 1,1,1,1,5", - "1,1,1,1,5"); - option_parser_register(opp, "-ptx_opcode_initiation_dp", OPT_CSTR, - &opcode_initiation_dp, - "Opcode initiation intervals for double precision " - "floating points " - "Default 8,8,8,8,130", - "8,8,8,8,130"); - option_parser_register(opp, "-ptx_opcode_initiation_sfu", OPT_CSTR, - &opcode_initiation_sfu, - "Opcode initiation intervals for sfu instructions" - "Default 8", - "8"); - option_parser_register(opp, "-ptx_opcode_initiation_tensor", OPT_CSTR, - &opcode_initiation_tensor, - "Opcode initiation intervals for tensor instructions" - "Default 64", - "64"); - option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str, - "CDP API latency " + "Default 1,1,19,25,145", + "1,1,19,25,145"); + option_parser_register(opp, "-ptx_opcode_latency_fp", OPT_CSTR, &opcode_latency_fp, + "Opcode latencies for single precision floating points " + "Default 1,1,1,1,30", + "1,1,1,1,30"); + option_parser_register(opp, "-ptx_opcode_latency_dp", OPT_CSTR, &opcode_latency_dp, + "Opcode latencies for double precision floating points " + "Default 8,8,8,8,335", + "8,8,8,8,335"); + option_parser_register(opp, "-ptx_opcode_latency_sfu", OPT_CSTR, &opcode_latency_sfu, + "Opcode latencies for SFU instructions" + "Default 8", + "8"); + option_parser_register(opp, "-ptx_opcode_latency_tesnor", OPT_CSTR, &opcode_latency_tensor, + "Opcode latencies for Tensor instructions" + "Default 64", + "64"); + option_parser_register(opp, "-ptx_opcode_initiation_int", OPT_CSTR, &opcode_initiation_int, + "Opcode initiation intervals for integers " + "Default 1,1,4,4,32", + "1,1,4,4,32"); + option_parser_register(opp, "-ptx_opcode_initiation_fp", OPT_CSTR, &opcode_initiation_fp, + "Opcode initiation intervals for single precision floating points " + "Default 1,1,1,1,5", + "1,1,1,1,5"); + option_parser_register(opp, "-ptx_opcode_initiation_dp", OPT_CSTR, &opcode_initiation_dp, + "Opcode initiation intervals for double precision floating points " + "Default 8,8,8,8,130", + "8,8,8,8,130"); + option_parser_register(opp, "-ptx_opcode_initiation_sfu", OPT_CSTR, &opcode_initiation_sfu, + "Opcode initiation intervals for sfu instructions" + "Default 8", + "8"); + option_parser_register(opp, "-ptx_opcode_initiation_tensor", OPT_CSTR, &opcode_initiation_tensor, + "Opcode initiation intervals for tensor instructions" + "Default 64", + "64"); + option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str, + "CDP API latency " - "Default 7200,8000,100,12000,1600", - "7200,8000,100,12000,1600"); -} - -void gpgpu_t::gpgpu_ptx_sim_bindNameToTexture( - const char *name, const struct textureReference *texref, int dim, - int readmode, int ext) { - std::string texname(name); - if (m_NameToTextureRef.find(texname) == m_NameToTextureRef.end()) { - m_NameToTextureRef[texname] = std::set(); - } else { - const struct textureReference *tr = *m_NameToTextureRef[texname].begin(); - assert(tr != NULL); - // asserts that all texrefs in set have same fields - assert(tr->normalized == texref->normalized && - tr->filterMode == texref->filterMode && - tr->addressMode[0] == texref->addressMode[0] && - tr->addressMode[1] == texref->addressMode[1] && - tr->addressMode[2] == texref->addressMode[2] && - tr->channelDesc.x == texref->channelDesc.x && - tr->channelDesc.y == texref->channelDesc.y && - tr->channelDesc.z == texref->channelDesc.z && - tr->channelDesc.w == texref->channelDesc.w && - tr->channelDesc.f == texref->channelDesc.f); - } - m_NameToTextureRef[texname].insert(texref); - m_TextureRefToName[texref] = texname; - const textureReferenceAttr *texAttr = new textureReferenceAttr( - texref, dim, (enum cudaTextureReadMode)readmode, ext); - m_NameToAttribute[texname] = texAttr; -} - -const char *gpgpu_t::gpgpu_ptx_sim_findNamefromTexture( - const struct textureReference *texref) { - std::map::const_iterator t = - m_TextureRefToName.find(texref); - assert(t != m_TextureRefToName.end()); - return t->second.c_str(); -} - -unsigned int intLOGB2(unsigned int v) { - unsigned int shift; - unsigned int r; - - r = 0; - - shift = ((v & 0xFFFF0000) != 0) << 4; - v >>= shift; - r |= shift; - shift = ((v & 0xFF00) != 0) << 3; - v >>= shift; - r |= shift; - shift = ((v & 0xF0) != 0) << 2; - v >>= shift; - r |= shift; - shift = ((v & 0xC) != 0) << 1; - v >>= shift; - r |= shift; - shift = ((v & 0x2) != 0) << 0; - v >>= shift; - r |= shift; - - return r; -} - -void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray( - const struct textureReference *texref, const struct cudaArray *array) { - std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); - - std::map::const_iterator t = - m_NameToCudaArray.find(texname); - // check that there's nothing there first - if (t != m_NameToCudaArray.end()) { - printf( - "GPGPU-Sim PTX: Warning: binding to texref associated with %s, which " - "was previously bound.\nImplicitly unbinding texref associated to %s " - "first\n", - texname.c_str(), texname.c_str()); - } - m_NameToCudaArray[texname] = array; - unsigned int texel_size_bits = - array->desc.w + array->desc.x + array->desc.y + array->desc.z; - unsigned int texel_size = texel_size_bits / 8; - unsigned int Tx, Ty; - int r; - - printf("GPGPU-Sim PTX: texel size = %d\n", texel_size); - printf("GPGPU-Sim PTX: texture cache linesize = %d\n", - m_function_model_config.get_texcache_linesize()); - // first determine base Tx size for given linesize - switch (m_function_model_config.get_texcache_linesize()) { - case 16: - Tx = 4; - break; - case 32: - Tx = 8; - break; - case 64: - Tx = 8; - break; - case 128: - Tx = 16; - break; - case 256: - Tx = 16; - break; - default: - printf( - "GPGPU-Sim PTX: Line size of %d bytes currently not supported.\n", - m_function_model_config.get_texcache_linesize()); + "Default 7200,8000,100,12000,1600", + "7200,8000,100,12000,1600"); +} + +void gpgpu_t::gpgpu_ptx_sim_bindNameToTexture(const char* name, const struct textureReference* texref, int dim, int readmode, int ext) +{ + std::string texname(name); + if (m_NameToTextureRef.find(texname)==m_NameToTextureRef.end()){ + m_NameToTextureRef[texname] = std::set(); + }else{ + const struct textureReference* tr = *m_NameToTextureRef[texname].begin(); + assert(tr!=NULL); + //asserts that all texrefs in set have same fields + assert(tr->normalized==texref->normalized&& + tr->filterMode==texref->filterMode&& + tr->addressMode[0]==texref->addressMode[0]&& + tr->addressMode[1]==texref->addressMode[1]&& + tr->addressMode[2]==texref->addressMode[2]&& + tr->channelDesc.x==texref->channelDesc.x&& + tr->channelDesc.y==texref->channelDesc.y&& + tr->channelDesc.z==texref->channelDesc.z&& + tr->channelDesc.w==texref->channelDesc.w&& + tr->channelDesc.f==texref->channelDesc.f + ); + } + m_NameToTextureRef[texname].insert(texref); + m_TextureRefToName[texref] = texname; + const textureReferenceAttr *texAttr = new textureReferenceAttr(texref, dim, (enum cudaTextureReadMode)readmode, ext); + m_NameToAttribute[texname] = texAttr; +} + +const char* gpgpu_t::gpgpu_ptx_sim_findNamefromTexture(const struct textureReference* texref) +{ + std::map::const_iterator t=m_TextureRefToName.find(texref); + assert( t != m_TextureRefToName.end() ); + return t->second.c_str(); +} + +unsigned int intLOGB2( unsigned int v ) { + unsigned int shift; + unsigned int r; + + r = 0; + + shift = (( v & 0xFFFF0000) != 0 ) << 4; v >>= shift; r |= shift; + shift = (( v & 0xFF00 ) != 0 ) << 3; v >>= shift; r |= shift; + shift = (( v & 0xF0 ) != 0 ) << 2; v >>= shift; r |= shift; + shift = (( v & 0xC ) != 0 ) << 1; v >>= shift; r |= shift; + shift = (( v & 0x2 ) != 0 ) << 0; v >>= shift; r |= shift; + + return r; +} + +void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* texref, const struct cudaArray* array) +{ + std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); + + std::map::const_iterator t=m_NameToCudaArray.find(texname); + //check that there's nothing there first + if(t != m_NameToCudaArray.end()){ + printf("GPGPU-Sim PTX: Warning: binding to texref associated with %s, which was previously bound.\nImplicitly unbinding texref associated to %s first\n", texname.c_str(), texname.c_str()); + } + m_NameToCudaArray[texname] = array; + unsigned int texel_size_bits = array->desc.w + array->desc.x + array->desc.y + array->desc.z; + unsigned int texel_size = texel_size_bits/8; + unsigned int Tx, Ty; + int r; + + printf("GPGPU-Sim PTX: texel size = %d\n", texel_size); + printf("GPGPU-Sim PTX: texture cache linesize = %d\n", m_function_model_config.get_texcache_linesize()); + //first determine base Tx size for given linesize + switch (m_function_model_config.get_texcache_linesize()) { + case 16: Tx = 4; break; + case 32: Tx = 8; break; + case 64: Tx = 8; break; + case 128: Tx = 16; break; + case 256: Tx = 16; break; + default: + printf("GPGPU-Sim PTX: Line size of %d bytes currently not supported.\n", m_function_model_config.get_texcache_linesize()); assert(0); break; - } - r = texel_size >> 2; - // modify base Tx size to take into account size of each texel in bytes - while (r != 0) { - Tx = Tx >> 1; - r = r >> 2; - } - // by now, got the correct Tx size, calculate correct Ty size - Ty = m_function_model_config.get_texcache_linesize() / (Tx * texel_size); - - printf( - "GPGPU-Sim PTX: Tx = %d; Ty = %d, Tx_numbits = %d, Ty_numbits = %d\n", - Tx, Ty, intLOGB2(Tx), intLOGB2(Ty)); - printf("GPGPU-Sim PTX: Texel size = %d bytes; texel_size_numbits = %d\n", - texel_size, intLOGB2(texel_size)); - printf( - "GPGPU-Sim PTX: Binding texture to array starting at devPtr32 = 0x%x\n", - array->devPtr32); - printf("GPGPU-Sim PTX: Texel size = %d bytes\n", texel_size); - struct textureInfo *texInfo = - (struct textureInfo *)malloc(sizeof(struct textureInfo)); - texInfo->Tx = Tx; - texInfo->Ty = Ty; - texInfo->Tx_numbits = intLOGB2(Tx); - texInfo->Ty_numbits = intLOGB2(Ty); - texInfo->texel_size = texel_size; - texInfo->texel_size_numbits = intLOGB2(texel_size); - m_NameToTextureInfo[texname] = texInfo; -} - -void gpgpu_t::gpgpu_ptx_sim_unbindTexture( - const struct textureReference *texref) { - // assumes bind-use-unbind-bind-use-unbind pattern - std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); - m_NameToCudaArray.erase(texname); - m_NameToTextureInfo.erase(texname); + } + r = texel_size >> 2; + //modify base Tx size to take into account size of each texel in bytes + while (r != 0) { + Tx = Tx >> 1; + r = r >> 2; + } + //by now, got the correct Tx size, calculate correct Ty size + Ty = m_function_model_config.get_texcache_linesize()/(Tx*texel_size); + + printf("GPGPU-Sim PTX: Tx = %d; Ty = %d, Tx_numbits = %d, Ty_numbits = %d\n", Tx, Ty, intLOGB2(Tx), intLOGB2(Ty)); + printf("GPGPU-Sim PTX: Texel size = %d bytes; texel_size_numbits = %d\n", texel_size, intLOGB2(texel_size)); + printf("GPGPU-Sim PTX: Binding texture to array starting at devPtr32 = 0x%x\n", array->devPtr32); + printf("GPGPU-Sim PTX: Texel size = %d bytes\n", texel_size); + struct textureInfo* texInfo = (struct textureInfo*) malloc(sizeof(struct textureInfo)); + texInfo->Tx = Tx; + texInfo->Ty = Ty; + texInfo->Tx_numbits = intLOGB2(Tx); + texInfo->Ty_numbits = intLOGB2(Ty); + texInfo->texel_size = texel_size; + texInfo->texel_size_numbits = intLOGB2(texel_size); + m_NameToTextureInfo[texname] = texInfo; +} + +void gpgpu_t::gpgpu_ptx_sim_unbindTexture(const struct textureReference* texref) +{ + //assumes bind-use-unbind-bind-use-unbind pattern + std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); + m_NameToCudaArray.erase(texname); + m_NameToTextureInfo.erase(texname); } #define MAX_INST_SIZE 8 /*bytes*/ -void function_info::ptx_assemble() { - if (m_assembled) { - return; - } - - // get the instructions into instruction memory... - unsigned num_inst = m_instructions.size(); - m_instr_mem_size = MAX_INST_SIZE * (num_inst + 1); - m_instr_mem = new ptx_instruction *[m_instr_mem_size]; - - printf("GPGPU-Sim PTX: instruction assembly for function \'%s\'... ", - m_name.c_str()); - fflush(stdout); - std::list::iterator i; - - addr_t PC = gpgpu_ctx->func_sim->g_assemble_code_next_pc; // globally unique - // address (across - // functions) - // start function on an aligned address - for (unsigned i = 0; i < (PC % MAX_INST_SIZE); i++) - gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction *)NULL); - PC += PC % MAX_INST_SIZE; - m_start_PC = PC; - - addr_t n = 0; // offset in m_instr_mem - // Why s_g_pc_to_insn.size() is needed to reserve additional memory for insts? - // reserve is cumulative. - // s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + - // MAX_INST_SIZE*m_instructions.size()); - gpgpu_ctx->s_g_pc_to_insn.reserve(MAX_INST_SIZE * m_instructions.size()); - for (i = m_instructions.begin(); i != m_instructions.end(); i++) { - ptx_instruction *pI = *i; - if (pI->is_label()) { - const symbol *l = pI->get_label(); - labels[l->name()] = n; - } else { - gpgpu_ctx->func_sim->g_pc_to_finfo[PC] = this; - m_instr_mem[n] = pI; - gpgpu_ctx->s_g_pc_to_insn.push_back(pI); - assert(pI == gpgpu_ctx->s_g_pc_to_insn[PC]); - pI->set_m_instr_mem_index(n); - pI->set_PC(PC); - assert(pI->inst_size() <= MAX_INST_SIZE); - for (unsigned i = 1; i < pI->inst_size(); i++) { - gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction *)NULL); - m_instr_mem[n + i] = NULL; +void function_info::ptx_assemble() +{ + if( m_assembled ) { + return; + } + + // get the instructions into instruction memory... + unsigned num_inst = m_instructions.size(); + m_instr_mem_size = MAX_INST_SIZE*(num_inst+1); + m_instr_mem = new ptx_instruction*[ m_instr_mem_size ]; + + printf("GPGPU-Sim PTX: instruction assembly for function \'%s\'... ", m_name.c_str() ); + fflush(stdout); + std::list::iterator i; + + addr_t PC = gpgpu_ctx->func_sim->g_assemble_code_next_pc; // globally unique address (across functions) + // start function on an aligned address + for( unsigned i=0; i < (PC%MAX_INST_SIZE); i++ ) + gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction*)NULL); + PC += PC%MAX_INST_SIZE; + m_start_PC = PC; + + addr_t n=0; // offset in m_instr_mem + //Why s_g_pc_to_insn.size() is needed to reserve additional memory for insts? reserve is cumulative. + //s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + MAX_INST_SIZE*m_instructions.size()); + gpgpu_ctx->s_g_pc_to_insn.reserve(MAX_INST_SIZE*m_instructions.size()); + for ( i=m_instructions.begin(); i != m_instructions.end(); i++ ) { + ptx_instruction *pI = *i; + if ( pI->is_label() ) { + const symbol *l = pI->get_label(); + labels[l->name()] = n; + } else { + gpgpu_ctx->func_sim->g_pc_to_finfo[PC] = this; + m_instr_mem[n] = pI; + gpgpu_ctx->s_g_pc_to_insn.push_back(pI); + assert(pI == gpgpu_ctx->s_g_pc_to_insn[PC]); + pI->set_m_instr_mem_index(n); + pI->set_PC(PC); + assert( pI->inst_size() <= MAX_INST_SIZE ); + for( unsigned i=1; i < pI->inst_size(); i++ ) { + gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction*)NULL); + m_instr_mem[n+i]=NULL; + } + n += pI->inst_size(); + PC += pI->inst_size(); } - n += pI->inst_size(); - PC += pI->inst_size(); - } - } - gpgpu_ctx->func_sim->g_assemble_code_next_pc = PC; - for (unsigned ii = 0; ii < n; - ii += m_instr_mem[ii]->inst_size()) { // handle branch instructions - ptx_instruction *pI = m_instr_mem[ii]; - if (pI->get_opcode() == BRA_OP || pI->get_opcode() == BREAKADDR_OP || - pI->get_opcode() == CALLP_OP) { - operand_info &target = pI->dst(); // get operand, e.g. target name - if (labels.find(target.name()) == labels.end()) { - printf( - "GPGPU-Sim PTX: Loader error (%s:%u): Branch label \"%s\" does not " - "appear in assembly code.", - pI->source_file(), pI->source_line(), target.name().c_str()); - abort(); + } + gpgpu_ctx->func_sim->g_assemble_code_next_pc=PC; + for ( unsigned ii=0; ii < n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions + ptx_instruction *pI = m_instr_mem[ii]; + if ( pI->get_opcode() == BRA_OP || pI->get_opcode() == BREAKADDR_OP || pI->get_opcode() == CALLP_OP) { + operand_info &target = pI->dst(); //get operand, e.g. target name + if ( labels.find(target.name()) == labels.end() ) { + printf("GPGPU-Sim PTX: Loader error (%s:%u): Branch label \"%s\" does not appear in assembly code.", + pI->source_file(),pI->source_line(), target.name().c_str() ); + abort(); + } + unsigned index = labels[ target.name() ]; //determine address from name + unsigned PC = m_instr_mem[index]->get_PC(); + m_symtab->set_label_address( target.get_symbol(), PC ); + target.set_type(label_t); } - unsigned index = labels[target.name()]; // determine address from name - unsigned PC = m_instr_mem[index]->get_PC(); - m_symtab->set_label_address(target.get_symbol(), PC); - target.set_type(label_t); - } - } - m_n = n; - printf(" done.\n"); - fflush(stdout); + } + m_n = n; + printf(" done.\n"); + fflush(stdout); -// disable pdom analysis here and do it at runtime + //disable pdom analysis here and do it at runtime #if 0 printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", m_name.c_str() ); create_basic_blocks(); @@ -383,2448 +323,2247 @@ void function_info::ptx_assemble() { #endif } -addr_t shared_to_generic(unsigned smid, addr_t addr) { - assert(addr < SHARED_MEM_SIZE_MAX); - return SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX + addr; -} - -addr_t global_to_generic(addr_t addr) { return addr; } +addr_t shared_to_generic( unsigned smid, addr_t addr ) +{ + assert( addr < SHARED_MEM_SIZE_MAX ); + return SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX + addr; +} -bool isspace_shared(unsigned smid, addr_t addr) { - addr_t start = SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX; - addr_t end = SHARED_GENERIC_START + (smid + 1) * SHARED_MEM_SIZE_MAX; - if ((addr >= end) || (addr < start)) return false; - return true; -} - -bool isspace_global(addr_t addr) { - return (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT); -} - -memory_space_t whichspace(addr_t addr) { - if ((addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT)) { - return global_space; - } else if (addr >= SHARED_GENERIC_START) { - return shared_space; - } else { - return local_space; - } -} - -addr_t generic_to_shared(unsigned smid, addr_t addr) { - assert(isspace_shared(smid, addr)); - return addr - (SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX); -} - -addr_t local_to_generic(unsigned smid, unsigned hwtid, addr_t addr) { - assert(addr < LOCAL_MEM_SIZE_MAX); - return LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + - (LOCAL_MEM_SIZE_MAX * hwtid) + addr; -} - -bool isspace_local(unsigned smid, unsigned hwtid, addr_t addr) { - addr_t start = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + - (LOCAL_MEM_SIZE_MAX * hwtid); - addr_t end = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + - (LOCAL_MEM_SIZE_MAX * (hwtid + 1)); - if ((addr >= end) || (addr < start)) return false; - return true; -} - -addr_t generic_to_local(unsigned smid, unsigned hwtid, addr_t addr) { - assert(isspace_local(smid, hwtid, addr)); - return addr - (LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + - (LOCAL_MEM_SIZE_MAX * hwtid)); -} - -addr_t generic_to_global(addr_t addr) { return addr; } - -void *gpgpu_t::gpu_malloc(size_t size) { - unsigned long long result = m_dev_malloc; - if (g_debug_execution >= 3) { - printf( - "GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address " - "0x%Lx\n", - size, m_dev_malloc); - fflush(stdout); - } - m_dev_malloc += size; - if (size % 256) - m_dev_malloc += (256 - size % 256); // align to 256 byte boundaries - return (void *)result; -} - -void *gpgpu_t::gpu_mallocarray(size_t size) { - unsigned long long result = m_dev_malloc; - if (g_debug_execution >= 3) { - printf( - "GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address " - "0x%Lx\n", - size, m_dev_malloc); - fflush(stdout); - } - m_dev_malloc += size; - if (size % 256) - m_dev_malloc += (256 - size % 256); // align to 256 byte boundaries - return (void *)result; -} - -void gpgpu_t::memcpy_to_gpu(size_t dst_start_addr, const void *src, - size_t count) { - if (g_debug_execution >= 3) { - printf( - "GPGPU-Sim PTX: copying %zu bytes from CPU[0x%Lx] to GPU[0x%Lx] ... ", - count, (unsigned long long)src, (unsigned long long)dst_start_addr); - fflush(stdout); - } - char *src_data = (char *)src; - for (unsigned n = 0; n < count; n++) - m_global_mem->write(dst_start_addr + n, 1, src_data + n, NULL, NULL); - - // Copy into the performance model. - // extern gpgpu_sim* g_the_gpu; - gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(dst_start_addr, count); - if (g_debug_execution >= 3) { - printf(" done.\n"); - fflush(stdout); - } -} - -void gpgpu_t::memcpy_from_gpu(void *dst, size_t src_start_addr, size_t count) { - if (g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to CPU[0x%Lx] ...", - count, (unsigned long long)src_start_addr, (unsigned long long)dst); - fflush(stdout); - } - unsigned char *dst_data = (unsigned char *)dst; - for (unsigned n = 0; n < count; n++) - m_global_mem->read(src_start_addr + n, 1, dst_data + n); - - // Copy into the performance model. - // extern gpgpu_sim* g_the_gpu; - gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(src_start_addr, count); - if (g_debug_execution >= 3) { - printf(" done.\n"); - fflush(stdout); - } -} - -void gpgpu_t::memcpy_gpu_to_gpu(size_t dst, size_t src, size_t count) { - if (g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to GPU[0x%Lx] ...", - count, (unsigned long long)src, (unsigned long long)dst); - fflush(stdout); - } - for (unsigned n = 0; n < count; n++) { - unsigned char tmp; - m_global_mem->read(src + n, 1, &tmp); - m_global_mem->write(dst + n, 1, &tmp, NULL, NULL); - } - if (g_debug_execution >= 3) { - printf(" done.\n"); - fflush(stdout); - } -} - -void gpgpu_t::gpu_memset(size_t dst_start_addr, int c, size_t count) { - if (g_debug_execution >= 3) { - printf( - "GPGPU-Sim PTX: setting %zu bytes of memory to 0x%x starting at " - "0x%Lx... ", - count, (unsigned char)c, (unsigned long long)dst_start_addr); - fflush(stdout); - } - unsigned char c_value = (unsigned char)c; - for (unsigned n = 0; n < count; n++) - m_global_mem->write(dst_start_addr + n, 1, &c_value, NULL, NULL); - if (g_debug_execution >= 3) { - printf(" done.\n"); - fflush(stdout); - } -} - -void cuda_sim::ptx_print_insn(address_type pc, FILE *fp) { - std::map::iterator f = g_pc_to_finfo.find(pc); - if (f == g_pc_to_finfo.end()) { - fprintf(fp, "", pc); - return; - } - function_info *finfo = f->second; - assert(finfo); - finfo->print_insn(pc, fp); -} - -std::string cuda_sim::ptx_get_insn_str(address_type pc) { - std::map::iterator f = g_pc_to_finfo.find(pc); - if (f == g_pc_to_finfo.end()) { -#define STR_SIZE 255 - char buff[STR_SIZE]; - buff[STR_SIZE - 1] = '\0'; - snprintf(buff, STR_SIZE, "", pc); - return std::string(buff); - } - function_info *finfo = f->second; - assert(finfo); - return finfo->get_insn_str(pc); -} - -void ptx_instruction::set_fp_or_int_archop() { - oprnd_type = UN_OP; - if ((m_opcode == MEMBAR_OP) || (m_opcode == SSY_OP) || (m_opcode == BRA_OP) || - (m_opcode == BAR_OP) || (m_opcode == RET_OP) || (m_opcode == RETP_OP) || - (m_opcode == NOP_OP) || (m_opcode == EXIT_OP) || (m_opcode == CALLP_OP) || - (m_opcode == CALL_OP)) { - // do nothing - } else if ((m_opcode == CVT_OP || m_opcode == SET_OP || - m_opcode == SLCT_OP)) { - if (get_type2() == F16_TYPE || get_type2() == F32_TYPE || - get_type2() == F64_TYPE || get_type2() == FF64_TYPE) { - oprnd_type = FP_OP; - } else - oprnd_type = INT_OP; - - } else { - if (get_type() == F16_TYPE || get_type() == F32_TYPE || - get_type() == F64_TYPE || get_type() == FF64_TYPE) { - oprnd_type = FP_OP; - } else - oprnd_type = INT_OP; - } -} -void ptx_instruction::set_mul_div_or_other_archop() { - sp_op = OTHER_OP; - if ((m_opcode != MEMBAR_OP) && (m_opcode != SSY_OP) && (m_opcode != BRA_OP) && - (m_opcode != BAR_OP) && (m_opcode != EXIT_OP) && (m_opcode != NOP_OP) && - (m_opcode != RETP_OP) && (m_opcode != RET_OP) && (m_opcode != CALLP_OP) && - (m_opcode != CALL_OP)) { - if (get_type() == F32_TYPE || get_type() == F64_TYPE || - get_type() == FF64_TYPE) { - switch (get_opcode()) { - case MUL_OP: - case MAD_OP: - sp_op = FP_MUL_OP; - break; - case DIV_OP: - sp_op = FP_DIV_OP; - break; - case LG2_OP: - sp_op = FP_LG_OP; - break; - case RSQRT_OP: - case SQRT_OP: - sp_op = FP_SQRT_OP; - break; - case RCP_OP: - sp_op = FP_DIV_OP; - break; - case SIN_OP: - case COS_OP: - sp_op = FP_SIN_OP; - break; - case EX2_OP: - sp_op = FP_EXP_OP; - break; - default: - if ((op == ALU_OP) || (op == TENSOR_CORE_OP)) sp_op = FP__OP; - break; - } - } else { - switch (get_opcode()) { - case MUL24_OP: - case MAD24_OP: - sp_op = INT_MUL24_OP; - break; - case MUL_OP: - case MAD_OP: - if (get_type() == U32_TYPE || get_type() == S32_TYPE || - get_type() == B32_TYPE) - sp_op = INT_MUL32_OP; - else - sp_op = INT_MUL_OP; - break; - case DIV_OP: - sp_op = INT_DIV_OP; - break; - default: - if ((op == ALU_OP)) sp_op = INT__OP; - break; - } - } - } -} - -void ptx_instruction::set_bar_type() { - if (m_opcode == BAR_OP) { - switch (m_barrier_op) { - case SYNC_OPTION: - bar_type = SYNC; - break; - case ARRIVE_OPTION: - bar_type = ARRIVE; - break; - case RED_OPTION: - bar_type = RED; - switch (m_atomic_spec) { - case ATOMIC_POPC: - red_type = POPC_RED; - break; - case ATOMIC_AND: - red_type = AND_RED; - break; - case ATOMIC_OR: - red_type = OR_RED; - break; - } - break; - default: - abort(); - } - } else if (m_opcode == SST_OP) { - bar_type = SYNC; - } -} - -void ptx_instruction::set_opcode_and_latency() { - unsigned int_latency[5]; - unsigned fp_latency[5]; - unsigned dp_latency[5]; - unsigned sfu_latency; - unsigned tensor_latency; - unsigned int_init[5]; - unsigned fp_init[5]; - unsigned dp_init[5]; - unsigned sfu_init; - unsigned tensor_init; - /* - * [0] ADD,SUB - * [1] MAX,Min - * [2] MUL - * [3] MAD - * [4] DIV - */ - sscanf(gpgpu_ctx->func_sim->opcode_latency_int, "%u,%u,%u,%u,%u", - &int_latency[0], &int_latency[1], &int_latency[2], &int_latency[3], - &int_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_fp, "%u,%u,%u,%u,%u", - &fp_latency[0], &fp_latency[1], &fp_latency[2], &fp_latency[3], - &fp_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_dp, "%u,%u,%u,%u,%u", - &dp_latency[0], &dp_latency[1], &dp_latency[2], &dp_latency[3], - &dp_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_sfu, "%u", &sfu_latency); - sscanf(gpgpu_ctx->func_sim->opcode_latency_tensor, "%u", &tensor_latency); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_int, "%u,%u,%u,%u,%u", - &int_init[0], &int_init[1], &int_init[2], &int_init[3], &int_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_fp, "%u,%u,%u,%u,%u", - &fp_init[0], &fp_init[1], &fp_init[2], &fp_init[3], &fp_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_dp, "%u,%u,%u,%u,%u", - &dp_init[0], &dp_init[1], &dp_init[2], &dp_init[3], &dp_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_sfu, "%u", &sfu_init); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_tensor, "%u", &tensor_init); - sscanf(gpgpu_ctx->func_sim->cdp_latency_str, "%u,%u,%u,%u,%u", - &gpgpu_ctx->func_sim->cdp_latency[0], - &gpgpu_ctx->func_sim->cdp_latency[1], - &gpgpu_ctx->func_sim->cdp_latency[2], - &gpgpu_ctx->func_sim->cdp_latency[3], - &gpgpu_ctx->func_sim->cdp_latency[4]); - - if (!m_operands.empty()) { - std::vector::iterator it; - for (it = ++m_operands.begin(); it != m_operands.end(); it++) { - num_operands++; - if ((it->is_reg() || it->is_vector())) { - num_regs++; - } - } - } - op = ALU_OP; - mem_op = NOT_TEX; - initiation_interval = latency = 1; - switch (m_opcode) { - case MOV_OP: - assert(!(has_memory_read() && has_memory_write())); - if (has_memory_read()) op = LOAD_OP; - if (has_memory_write()) op = STORE_OP; - break; - case LD_OP: - op = LOAD_OP; - break; - case MMA_LD_OP: - op = TENSOR_CORE_LOAD_OP; - break; - case LDU_OP: - op = LOAD_OP; - break; - case ST_OP: - op = STORE_OP; - break; - case MMA_ST_OP: - op = TENSOR_CORE_STORE_OP; - break; - case BRA_OP: - op = BRANCH_OP; - break; - case BREAKADDR_OP: - op = BRANCH_OP; - break; - case TEX_OP: - op = LOAD_OP; - mem_op = TEX; - break; - case ATOM_OP: - op = LOAD_OP; - break; - case BAR_OP: - op = BARRIER_OP; - break; - case SST_OP: - op = BARRIER_OP; - break; - case MEMBAR_OP: - op = MEMORY_BARRIER_OP; - break; - case CALL_OP: { - if (m_is_printf || m_is_cdp) { - op = ALU_OP; - } else - op = CALL_OPS; - break; - } - case CALLP_OP: { - if (m_is_printf || m_is_cdp) { - op = ALU_OP; - } else - op = CALL_OPS; - break; - } - case RET_OP: - case RETP_OP: - op = RET_OPS; - break; - case ADD_OP: - case ADDP_OP: - case ADDC_OP: - case SUB_OP: - case SUBC_OP: - // ADD,SUB latency - switch (get_type()) { - case F32_TYPE: - latency = fp_latency[0]; - initiation_interval = fp_init[0]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[0]; - initiation_interval = dp_init[0]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: // Use int settings for default - latency = int_latency[0]; - initiation_interval = int_init[0]; - op = INTP_OP; - break; - } - break; - case MAX_OP: - case MIN_OP: - // MAX,MIN latency - switch (get_type()) { - case F32_TYPE: - latency = fp_latency[1]; - initiation_interval = fp_init[1]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[1]; - initiation_interval = dp_init[1]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: // Use int settings for default - latency = int_latency[1]; - initiation_interval = int_init[1]; - op = INTP_OP; - break; - } - break; - case MUL_OP: - // MUL latency - switch (get_type()) { - case F32_TYPE: - latency = fp_latency[2]; - initiation_interval = fp_init[2]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[2]; - initiation_interval = dp_init[2]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: // Use int settings for default - latency = int_latency[2]; - initiation_interval = int_init[2]; - op = INTP_OP; - break; - } - break; - case MAD_OP: - case MADC_OP: - case MADP_OP: - // MAD latency - switch (get_type()) { - case F32_TYPE: - latency = fp_latency[3]; - initiation_interval = fp_init[3]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[3]; - initiation_interval = dp_init[3]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: // Use int settings for default - latency = int_latency[3]; - initiation_interval = int_init[3]; - op = INTP_OP; - break; - } - break; - case DIV_OP: - // Floating point only - op = SFU_OP; - switch (get_type()) { - case F32_TYPE: - latency = fp_latency[4]; - initiation_interval = fp_init[4]; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[4]; - initiation_interval = dp_init[4]; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: // Use int settings for default - latency = int_latency[4]; - initiation_interval = int_init[4]; - break; - } - break; - case SQRT_OP: - case SIN_OP: - case COS_OP: - case EX2_OP: - case LG2_OP: - case RSQRT_OP: - case RCP_OP: - latency = sfu_latency; - initiation_interval = sfu_init; - op = SFU_OP; - break; - case MMA_OP: - latency = tensor_latency; - initiation_interval = tensor_init; - op = TENSOR_CORE_OP; - break; - case SHFL_OP: - latency = 4; - initiation_interval = 4; - break; - default: - break; - } - set_fp_or_int_archop(); - set_mul_div_or_other_archop(); +addr_t global_to_generic( addr_t addr ) +{ + return addr; } -void ptx_thread_info::ptx_fetch_inst(inst_t &inst) const { - addr_t pc = get_pc(); - const ptx_instruction *pI = m_func_info->get_instruction(pc); - inst = (const inst_t &)*pI; - assert(inst.valid()); +bool isspace_shared( unsigned smid, addr_t addr ) +{ + addr_t start = SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX; + addr_t end = SHARED_GENERIC_START + (smid+1)*SHARED_MEM_SIZE_MAX; + if( (addr >= end) || (addr < start) ) + return false; + return true; } -static unsigned datatype2size(unsigned data_type) { - unsigned data_size; - switch (data_type) { - case B8_TYPE: - case S8_TYPE: - case U8_TYPE: - data_size = 1; - break; - case B16_TYPE: - case S16_TYPE: - case U16_TYPE: - case F16_TYPE: - data_size = 2; - break; - case B32_TYPE: - case S32_TYPE: - case U32_TYPE: - case F32_TYPE: - data_size = 4; - break; - case B64_TYPE: - case BB64_TYPE: - case S64_TYPE: - case U64_TYPE: - case F64_TYPE: - case FF64_TYPE: - data_size = 8; - break; - case BB128_TYPE: - data_size = 16; - break; - default: - assert(0); +bool isspace_global( addr_t addr ) +{ + return (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT); +} + +memory_space_t whichspace( addr_t addr ) +{ + if( (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT) ) { + return global_space; + } else if( addr >= SHARED_GENERIC_START ) { + return shared_space; + } else { + return local_space; + } +} + +addr_t generic_to_shared( unsigned smid, addr_t addr ) +{ + assert(isspace_shared(smid,addr)); + return addr - (SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX); +} + +addr_t local_to_generic( unsigned smid, unsigned hwtid, addr_t addr ) +{ + assert(addr < LOCAL_MEM_SIZE_MAX); + return LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid) + addr; +} + +bool isspace_local( unsigned smid, unsigned hwtid, addr_t addr ) +{ + addr_t start = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid); + addr_t end = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * (hwtid+1)); + if( (addr >= end) || (addr < start) ) + return false; + return true; +} + +addr_t generic_to_local( unsigned smid, unsigned hwtid, addr_t addr ) +{ + assert(isspace_local(smid,hwtid,addr)); + return addr - (LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid)); +} + +addr_t generic_to_global( addr_t addr ) +{ + return addr; +} + + +void* gpgpu_t::gpu_malloc( size_t size ) +{ + unsigned long long result = m_dev_malloc; + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address 0x%Lx\n", size, m_dev_malloc ); + fflush(stdout); + } + m_dev_malloc += size; + if (size%256) m_dev_malloc += (256 - size%256); //align to 256 byte boundaries + return(void*) result; +} + +void* gpgpu_t::gpu_mallocarray( size_t size ) +{ + unsigned long long result = m_dev_malloc; + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address 0x%Lx\n", size, m_dev_malloc ); + fflush(stdout); + } + m_dev_malloc += size; + if (size%256) m_dev_malloc += (256 - size%256); //align to 256 byte boundaries + return(void*) result; +} + + +void gpgpu_t::memcpy_to_gpu( size_t dst_start_addr, const void *src, size_t count ) +{ + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: copying %zu bytes from CPU[0x%Lx] to GPU[0x%Lx] ... ", count, (unsigned long long) src, (unsigned long long) dst_start_addr ); + fflush(stdout); + } + char *src_data = (char*)src; + for (unsigned n=0; n < count; n ++ ) + m_global_mem->write(dst_start_addr+n,1, src_data+n,NULL,NULL); + + // Copy into the performance model. + //extern gpgpu_sim* g_the_gpu; + gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(dst_start_addr, count); + if(g_debug_execution >= 3) { + printf( " done.\n"); + fflush(stdout); + } +} + +void gpgpu_t::memcpy_from_gpu( void *dst, size_t src_start_addr, size_t count ) +{ + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to CPU[0x%Lx] ...", count, (unsigned long long) src_start_addr, (unsigned long long) dst ); + fflush(stdout); + } + unsigned char *dst_data = (unsigned char*)dst; + for (unsigned n=0; n < count; n ++ ) + m_global_mem->read(src_start_addr+n,1,dst_data+n); + + // Copy into the performance model. + //extern gpgpu_sim* g_the_gpu; + gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(src_start_addr, count); + if(g_debug_execution >= 3) { + printf( " done.\n"); + fflush(stdout); + } +} + +void gpgpu_t::memcpy_gpu_to_gpu( size_t dst, size_t src, size_t count ) +{ + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to GPU[0x%Lx] ...", count, + (unsigned long long) src, (unsigned long long) dst ); + fflush(stdout); + } + for (unsigned n=0; n < count; n ++ ) { + unsigned char tmp; + m_global_mem->read(src+n,1,&tmp); + m_global_mem->write(dst+n,1, &tmp,NULL,NULL); + } + if(g_debug_execution >= 3) { + printf( " done.\n"); + fflush(stdout); + } +} + +void gpgpu_t::gpu_memset( size_t dst_start_addr, int c, size_t count ) +{ + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: setting %zu bytes of memory to 0x%x starting at 0x%Lx... ", + count, (unsigned char) c, (unsigned long long) dst_start_addr ); + fflush(stdout); + } + unsigned char c_value = (unsigned char)c; + for (unsigned n=0; n < count; n ++ ) + m_global_mem->write(dst_start_addr+n,1,&c_value,NULL,NULL); + if(g_debug_execution >= 3) { + printf( " done.\n"); + fflush(stdout); + } +} + +void cuda_sim::ptx_print_insn( address_type pc, FILE *fp ) +{ + std::map::iterator f = g_pc_to_finfo.find(pc); + if( f == g_pc_to_finfo.end() ) { + fprintf(fp,"", pc ); + return; + } + function_info *finfo = f->second; + assert( finfo ); + finfo->print_insn(pc,fp); +} + +std::string cuda_sim::ptx_get_insn_str( address_type pc ) +{ + std::map::iterator f = g_pc_to_finfo.find(pc); + if( f == g_pc_to_finfo.end() ) { + #define STR_SIZE 255 + char buff[STR_SIZE]; + buff[STR_SIZE - 1] = '\0'; + snprintf(buff, STR_SIZE,"", pc ); + return std::string(buff); + } + function_info *finfo = f->second; + assert( finfo ); + return finfo->get_insn_str(pc); +} + +void ptx_instruction::set_fp_or_int_archop(){ + oprnd_type=UN_OP; + if((m_opcode == MEMBAR_OP)||(m_opcode == SSY_OP )||(m_opcode == BRA_OP) || (m_opcode == BAR_OP) || (m_opcode == RET_OP) || (m_opcode == RETP_OP) || (m_opcode == NOP_OP) || (m_opcode == EXIT_OP) || (m_opcode == CALLP_OP) || (m_opcode == CALL_OP)){ + // do nothing + }else if((m_opcode == CVT_OP || m_opcode == SET_OP || m_opcode == SLCT_OP)){ + if(get_type2()==F16_TYPE || get_type2()==F32_TYPE || get_type2() == F64_TYPE || get_type2() == FF64_TYPE){ + oprnd_type= FP_OP; + }else oprnd_type=INT_OP; + + }else{ + if(get_type()==F16_TYPE || get_type()==F32_TYPE || get_type() == F64_TYPE || get_type() == FF64_TYPE){ + oprnd_type= FP_OP; + }else oprnd_type=INT_OP; + } +} +void ptx_instruction::set_mul_div_or_other_archop(){ + sp_op=OTHER_OP; + if((m_opcode != MEMBAR_OP) && (m_opcode != SSY_OP) && (m_opcode != BRA_OP) && (m_opcode != BAR_OP) && (m_opcode != EXIT_OP) && (m_opcode != NOP_OP) && (m_opcode != RETP_OP) && (m_opcode != RET_OP) && (m_opcode != CALLP_OP) && (m_opcode != CALL_OP)){ + if(get_type()==F32_TYPE || get_type() == F64_TYPE || get_type() == FF64_TYPE){ + switch(get_opcode()){ + case MUL_OP: + case MAD_OP: + sp_op=FP_MUL_OP; + break; + case DIV_OP: + sp_op=FP_DIV_OP; + break; + case LG2_OP: + sp_op=FP_LG_OP; + break; + case RSQRT_OP: + case SQRT_OP: + sp_op=FP_SQRT_OP; + break; + case RCP_OP: + sp_op=FP_DIV_OP; + break; + case SIN_OP: + case COS_OP: + sp_op=FP_SIN_OP; + break; + case EX2_OP: + sp_op=FP_EXP_OP; + break; + default: + if((op==ALU_OP)||(op==TENSOR_CORE_OP)) + sp_op=FP__OP; + break; + + } + }else { + switch(get_opcode()){ + case MUL24_OP: + case MAD24_OP: + sp_op=INT_MUL24_OP; + break; + case MUL_OP: + case MAD_OP: + if(get_type()==U32_TYPE || get_type()==S32_TYPE || get_type()==B32_TYPE) + sp_op=INT_MUL32_OP; + else + sp_op=INT_MUL_OP; + break; + case DIV_OP: + sp_op=INT_DIV_OP; + break; + default: + if((op==ALU_OP)) + sp_op=INT__OP; + break; + } + } + } + +} + + + +void ptx_instruction::set_bar_type() +{ + if(m_opcode==BAR_OP) { + switch(m_barrier_op){ + case SYNC_OPTION: + bar_type = SYNC; + break; + case ARRIVE_OPTION: + bar_type = ARRIVE; + break; + case RED_OPTION: + bar_type = RED; + switch(m_atomic_spec){ + case ATOMIC_POPC: + red_type = POPC_RED; + break; + case ATOMIC_AND: + red_type = AND_RED; + break; + case ATOMIC_OR: + red_type = OR_RED; + break; + } + break; + default: + abort(); + } + } + else if(m_opcode==SST_OP) { + bar_type = SYNC; + } +} + + +void ptx_instruction::set_opcode_and_latency() +{ + unsigned int_latency[5]; + unsigned fp_latency[5]; + unsigned dp_latency[5]; + unsigned sfu_latency; + unsigned tensor_latency; + unsigned int_init[5]; + unsigned fp_init[5]; + unsigned dp_init[5]; + unsigned sfu_init; + unsigned tensor_init; + /* + * [0] ADD,SUB + * [1] MAX,Min + * [2] MUL + * [3] MAD + * [4] DIV + */ + sscanf(gpgpu_ctx->func_sim->opcode_latency_int, "%u,%u,%u,%u,%u", + &int_latency[0],&int_latency[1],&int_latency[2], + &int_latency[3],&int_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_fp, "%u,%u,%u,%u,%u", + &fp_latency[0],&fp_latency[1],&fp_latency[2], + &fp_latency[3],&fp_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_dp, "%u,%u,%u,%u,%u", + &dp_latency[0],&dp_latency[1],&dp_latency[2], + &dp_latency[3],&dp_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_sfu, "%u", + &sfu_latency); + sscanf(gpgpu_ctx->func_sim->opcode_latency_tensor, "%u", + &tensor_latency); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_int, "%u,%u,%u,%u,%u", + &int_init[0],&int_init[1],&int_init[2], + &int_init[3],&int_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_fp, "%u,%u,%u,%u,%u", + &fp_init[0],&fp_init[1],&fp_init[2], + &fp_init[3],&fp_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_dp, "%u,%u,%u,%u,%u", + &dp_init[0],&dp_init[1],&dp_init[2], + &dp_init[3],&dp_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_sfu, "%u", + &sfu_init); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_tensor, "%u", + &tensor_init); + sscanf(gpgpu_ctx->func_sim->cdp_latency_str, "%u,%u,%u,%u,%u", + &gpgpu_ctx->func_sim->cdp_latency[0], + &gpgpu_ctx->func_sim->cdp_latency[1], + &gpgpu_ctx->func_sim->cdp_latency[2], + &gpgpu_ctx->func_sim->cdp_latency[3], + &gpgpu_ctx->func_sim->cdp_latency[4]); + + if(!m_operands.empty()){ + std::vector::iterator it; + for(it=++m_operands.begin();it!=m_operands.end();it++){ + num_operands++; + if((it->is_reg() || it->is_vector())){ + num_regs++; + } + } + } + op = ALU_OP; + mem_op= NOT_TEX; + initiation_interval = latency = 1; + switch( m_opcode ) { + case MOV_OP: + assert( !(has_memory_read() && has_memory_write()) ); + if ( has_memory_read() ) op = LOAD_OP; + if ( has_memory_write() ) op = STORE_OP; + break; + case LD_OP: op = LOAD_OP; break; + case MMA_LD_OP: op = TENSOR_CORE_LOAD_OP; break; + case LDU_OP: op = LOAD_OP; break; + case ST_OP: op = STORE_OP; break; + case MMA_ST_OP: op = TENSOR_CORE_STORE_OP; break; + case BRA_OP: op = BRANCH_OP; break; + case BREAKADDR_OP: op = BRANCH_OP; break; + case TEX_OP: op = LOAD_OP; mem_op=TEX; break; + case ATOM_OP: op = LOAD_OP; break; + case BAR_OP: op = BARRIER_OP; break; + case SST_OP: op = BARRIER_OP; break; + case MEMBAR_OP: op = MEMORY_BARRIER_OP; break; + case CALL_OP: + { + if(m_is_printf || m_is_cdp) { + op = ALU_OP; + } + else + op = CALL_OPS; + break; + } + case CALLP_OP: + { + if(m_is_printf || m_is_cdp) { + op = ALU_OP; + } + else + op = CALL_OPS; + break; + } + case RET_OP: case RETP_OP: op = RET_OPS;break; + case ADD_OP: case ADDP_OP: case ADDC_OP: case SUB_OP: case SUBC_OP: + //ADD,SUB latency + switch(get_type()){ + case F32_TYPE: + latency = fp_latency[0]; + initiation_interval = fp_init[0]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[0]; + initiation_interval = dp_init[0]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: //Use int settings for default + latency = int_latency[0]; + initiation_interval = int_init[0]; + op = INTP_OP; + break; + } + break; + case MAX_OP: case MIN_OP: + //MAX,MIN latency + switch(get_type()){ + case F32_TYPE: + latency = fp_latency[1]; + initiation_interval = fp_init[1]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[1]; + initiation_interval = dp_init[1]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: //Use int settings for default + latency = int_latency[1]; + initiation_interval = int_init[1]; + op = INTP_OP; + break; + } + break; + case MUL_OP: + //MUL latency + switch(get_type()){ + case F32_TYPE: + latency = fp_latency[2]; + initiation_interval = fp_init[2]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[2]; + initiation_interval = dp_init[2]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: //Use int settings for default + latency = int_latency[2]; + initiation_interval = int_init[2]; + op = INTP_OP; + break; + } + break; + case MAD_OP: case MADC_OP: case MADP_OP: + //MAD latency + switch(get_type()){ + case F32_TYPE: + latency = fp_latency[3]; + initiation_interval = fp_init[3]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[3]; + initiation_interval = dp_init[3]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: //Use int settings for default + latency = int_latency[3]; + initiation_interval = int_init[3]; + op = INTP_OP; + break; + } + break; + case DIV_OP: + // Floating point only + op = SFU_OP; + switch(get_type()){ + case F32_TYPE: + latency = fp_latency[4]; + initiation_interval = fp_init[4]; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[4]; + initiation_interval = dp_init[4]; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: //Use int settings for default + latency = int_latency[4]; + initiation_interval = int_init[4]; + break; + } + break; + case SQRT_OP: case SIN_OP: case COS_OP: case EX2_OP: case LG2_OP: case RSQRT_OP: case RCP_OP: + latency = sfu_latency; + initiation_interval = sfu_init; + op = SFU_OP; break; - } - return data_size; -} - -void ptx_instruction::pre_decode() { - pc = m_PC; - isize = m_inst_size; - for (unsigned i = 0; i < MAX_OUTPUT_VALUES; i++) { - out[i] = 0; - } - for (unsigned i = 0; i < MAX_INPUT_VALUES; i++) { - in[i] = 0; - } - incount = 0; - outcount = 0; - is_vectorin = 0; - is_vectorout = 0; - std::fill_n(arch_reg.src, MAX_REG_OPERANDS, -1); - std::fill_n(arch_reg.dst, MAX_REG_OPERANDS, -1); - pred = 0; - ar1 = 0; - ar2 = 0; - space = m_space_spec; - memory_op = no_memory_op; - data_size = 0; - if (has_memory_read() || has_memory_write()) { - unsigned to_type = get_type(); - data_size = datatype2size(to_type); - memory_op = has_memory_read() ? memory_load : memory_store; - } - - bool has_dst = false; - - switch (get_opcode()) { -#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ - case OP: \ - has_dst = (DST != 0); \ - break; -#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ - case OP: \ - has_dst = (DST != 0); \ - break; + case MMA_OP: + latency = tensor_latency; + initiation_interval = tensor_init; + op=TENSOR_CORE_OP; + break; + case SHFL_OP: + latency = 4; + initiation_interval = 4; + break; + default: + break; + } + set_fp_or_int_archop(); + set_mul_div_or_other_archop(); + +} + +void ptx_thread_info::ptx_fetch_inst( inst_t &inst ) const +{ + addr_t pc = get_pc(); + const ptx_instruction *pI = m_func_info->get_instruction(pc); + inst = (const inst_t&)*pI; + assert( inst.valid() ); +} + +static unsigned datatype2size( unsigned data_type ) +{ + unsigned data_size; + switch ( data_type ) { + case B8_TYPE: + case S8_TYPE: + case U8_TYPE: + data_size = 1; break; + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: + case F16_TYPE: + data_size = 2; break; + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: + case F32_TYPE: + data_size = 4; break; + case B64_TYPE: + case BB64_TYPE: + case S64_TYPE: + case U64_TYPE: + case F64_TYPE: + case FF64_TYPE: + data_size = 8; break; + case BB128_TYPE: + data_size = 16; break; + default: assert(0); break; + } + return data_size; +} + +void ptx_instruction::pre_decode() +{ + pc = m_PC; + isize = m_inst_size; + for(unsigned i=0; i= 1) out[0] = o.reg1_num(); - if (num_elem >= 2) out[1] = o.reg2_num(); - if (num_elem >= 3) out[2] = o.reg3_num(); - if (num_elem >= 4) out[3] = o.reg4_num(); - if (num_elem >= 5) out[4] = o.reg5_num(); - if (num_elem >= 6) out[5] = o.reg6_num(); - if (num_elem >= 7) out[6] = o.reg7_num(); - if (num_elem >= 8) out[7] = o.reg8_num(); - for (int i = 0; i < num_elem; i++) arch_reg.dst[i] = o.arch_reg_num(i); - } - } else { - if (o.is_reg() && !o.is_non_arch_reg()) { - int reg_num = o.reg_num(); - arch_reg.src[m] = o.arch_reg_num(); - switch (m) { - case 0: - in[0] = reg_num; - break; - case 1: - in[1] = reg_num; - break; - case 2: - in[2] = reg_num; - break; - default: - break; - } - m++; - } else if (o.is_vector()) { - // assert(m == 0); //only support 1 vector operand (for textures) right - // now - is_vectorout = 1; - unsigned num_elem = o.get_vect_nelem(); - if (num_elem >= 1) in[m + 0] = o.reg1_num(); - if (num_elem >= 2) in[m + 1] = o.reg2_num(); - if (num_elem >= 3) in[m + 2] = o.reg3_num(); - if (num_elem >= 4) in[m + 3] = o.reg4_num(); - if (num_elem >= 5) in[m + 4] = o.reg5_num(); - if (num_elem >= 6) in[m + 5] = o.reg6_num(); - if (num_elem >= 7) in[m + 6] = o.reg7_num(); - if (num_elem >= 8) in[m + 7] = o.reg8_num(); - for (int i = 0; i < num_elem; i++) - arch_reg.src[m + i] = o.arch_reg_num(i); - m += num_elem; + } + + set_opcode_and_latency(); + set_bar_type(); + // Get register operands + int n=0,m=0; + ptx_instruction::const_iterator opr=op_iter_begin(); + for ( ; opr != op_iter_end(); opr++, n++ ) { //process operands + const operand_info &o = *opr; + if ( has_dst && n==0 ) { + // Do not set the null register "_" as an architectural register + if ( o.is_reg() && !o.is_non_arch_reg() ) { + out[0] = o.reg_num(); + arch_reg.dst[0] = o.arch_reg_num(); + } else if ( o.is_vector() ) { + is_vectorin = 1; + unsigned num_elem = o.get_vect_nelem(); + if( num_elem >= 1 ) out[0] = o.reg1_num(); + if( num_elem >= 2 ) out[1] = o.reg2_num(); + if( num_elem >= 3 ) out[2] = o.reg3_num(); + if( num_elem >= 4 ) out[3] = o.reg4_num(); + if( num_elem >= 5 ) out[4] = o.reg5_num(); + if( num_elem >= 6 ) out[5] = o.reg6_num(); + if( num_elem >= 7 ) out[6] = o.reg7_num(); + if( num_elem >= 8 ) out[7] = o.reg8_num(); + for (int i = 0; i < num_elem; i++) + arch_reg.dst[i] = o.arch_reg_num(i); + } + } else { + if ( o.is_reg() && !o.is_non_arch_reg() ) { + int reg_num = o.reg_num(); + arch_reg.src[m] = o.arch_reg_num(); + switch ( m ) { + case 0: in[0] = reg_num; break; + case 1: in[1] = reg_num; break; + case 2: in[2] = reg_num; break; + default: break; + } + m++; + } else if ( o.is_vector() ) { + //assert(m == 0); //only support 1 vector operand (for textures) right now + is_vectorout = 1; + unsigned num_elem = o.get_vect_nelem(); + if( num_elem >= 1 ) in[m+0] = o.reg1_num(); + if( num_elem >= 2 ) in[m+1] = o.reg2_num(); + if( num_elem >= 3 ) in[m+2] = o.reg3_num(); + if( num_elem >= 4 ) in[m+3] = o.reg4_num(); + if( num_elem >= 5 ) in[m+4] = o.reg5_num(); + if( num_elem >= 6 ) in[m+5] = o.reg6_num(); + if( num_elem >= 7 ) in[m+6] = o.reg7_num(); + if( num_elem >= 8 ) in[m+7] = o.reg8_num(); + for (int i = 0; i < num_elem; i++) + arch_reg.src[m+i] = o.arch_reg_num(i); + m+=num_elem; + } } - } - } - - // Setting number of input and output operands which is required for - // scoreboard check - for (int i = 0; i < MAX_OUTPUT_VALUES; i++) - if (out[i] > 0) outcount++; - - for (int i = 0; i < MAX_INPUT_VALUES; i++) - if (in[i] > 0) incount++; - - // Get predicate - if (has_pred()) { - const operand_info &p = get_pred(); - pred = p.reg_num(); - } - - // Get address registers inside memory operands. - // Assuming only one memory operand per instruction, - // and maximum of two address registers for one memory operand. - if (has_memory_read() || has_memory_write()) { - ptx_instruction::const_iterator op = op_iter_begin(); - for (; op != op_iter_end(); op++, n++) { // process operands - const operand_info &o = *op; - - if (o.is_memory_operand()) { - // We do not support the null register as a memory operand - assert(!o.is_non_arch_reg()); - - // Check PTXPlus-type operand - // memory operand with addressing (ex. s[0x4] or g[$r1]) - if (o.is_memory_operand2()) { - // memory operand with one address register (ex. g[$r1+0x4] or - // s[$r2+=0x4]) - if (o.get_double_operand_type() == 0 || - o.get_double_operand_type() == 3) { - ar1 = o.reg_num(); - arch_reg.src[4] = o.arch_reg_num(); - // TODO: address register in $r2+=0x4 should be an output register - // as well - } - // memory operand with two address register (ex. s[$r1+$r1] or - // g[$r1+=$r2]) - else if (o.get_double_operand_type() == 1 || - o.get_double_operand_type() == 2) { - ar1 = o.reg1_num(); - arch_reg.src[4] = o.arch_reg_num(); - ar2 = o.reg2_num(); - arch_reg.src[5] = o.arch_reg_num(); - // TODO: first address register in $r1+=$r2 should be an output - // register as well - } - } else if (o.is_immediate_address()) { - } - // Regular PTX operand - else if (o.get_symbol() - ->type() - ->get_key() - .is_reg()) { // Memory operand contains a register - ar1 = o.reg_num(); - arch_reg.src[4] = o.arch_reg_num(); - } + } + + //Setting number of input and output operands which is required for scoreboard check + for(int i=0;i0) + outcount++; + + for(int i=0;i0) + incount++; + + // Get predicate + if(has_pred()) { + const operand_info &p = get_pred(); + pred = p.reg_num(); + } + + // Get address registers inside memory operands. + // Assuming only one memory operand per instruction, + // and maximum of two address registers for one memory operand. + if( has_memory_read() || has_memory_write() ) { + ptx_instruction::const_iterator op=op_iter_begin(); + for ( ; op != op_iter_end(); op++, n++ ) { //process operands + const operand_info &o = *op; + + if(o.is_memory_operand()) { + // We do not support the null register as a memory operand + assert( !o.is_non_arch_reg() ); + + // Check PTXPlus-type operand + // memory operand with addressing (ex. s[0x4] or g[$r1]) + if(o.is_memory_operand2()) { + + // memory operand with one address register (ex. g[$r1+0x4] or s[$r2+=0x4]) + if(o.get_double_operand_type() == 0 || o.get_double_operand_type() == 3){ + ar1 = o.reg_num(); + arch_reg.src[4] = o.arch_reg_num(); + // TODO: address register in $r2+=0x4 should be an output register as well + } + // memory operand with two address register (ex. s[$r1+$r1] or g[$r1+=$r2]) + else if(o.get_double_operand_type() == 1 || o.get_double_operand_type() == 2) { + ar1 = o.reg1_num(); + arch_reg.src[4] = o.arch_reg_num(); + ar2 = o.reg2_num(); + arch_reg.src[5] = o.arch_reg_num(); + // TODO: first address register in $r1+=$r2 should be an output register as well + } + } + else if(o.is_immediate_address()){ + + } + // Regular PTX operand + else if (o.get_symbol()->type()->get_key().is_reg()) { // Memory operand contains a register + ar1 = o.reg_num(); + arch_reg.src[4] = o.arch_reg_num(); + } + + } } - } - } - - // get reconvergence pc - reconvergence_pc = gpgpu_ctx->func_sim->get_converge_point(pc); - - m_decoded = true; -} - -void function_info::add_param_name_type_size(unsigned index, std::string name, - int type, size_t size, bool ptr, - memory_space_t space) { - unsigned parsed_index; - char buffer[2048]; - snprintf(buffer, 2048, "%s_param_%%u", m_name.c_str()); - int ntokens = sscanf(name.c_str(), buffer, &parsed_index); - if (ntokens == 1) { - assert(m_ptx_kernel_param_info.find(parsed_index) == - m_ptx_kernel_param_info.end()); - m_ptx_kernel_param_info[parsed_index] = - param_info(name, type, size, ptr, space); - } else { - assert(m_ptx_kernel_param_info.find(index) == - m_ptx_kernel_param_info.end()); - m_ptx_kernel_param_info[index] = param_info(name, type, size, ptr, space); - } -} - -void function_info::add_param_data(unsigned argn, - struct gpgpu_ptx_sim_arg *args) { - const void *data = args->m_start; - - bool scratchpad_memory_param = - false; // Is this parameter in CUDA shared memory or OpenCL local memory - - std::map::iterator i = - m_ptx_kernel_param_info.find(argn); - if (i != m_ptx_kernel_param_info.end()) { - if (i->second.is_ptr_shared()) { - assert( - args->m_start == NULL && - "OpenCL parameter pointer to local memory must have NULL as value"); - scratchpad_memory_param = true; - } else { - param_t tmp; - tmp.pdata = args->m_start; - tmp.size = args->m_nbytes; - tmp.offset = args->m_offset; - tmp.type = 0; - i->second.add_data(tmp); - i->second.add_offset((unsigned)args->m_offset); - } - } else { - scratchpad_memory_param = true; - } - - if (scratchpad_memory_param) { - // This should only happen for OpenCL: - // - // The LLVM PTX compiler in NVIDIA's driver (version 190.29) - // does not generate an argument in the function declaration - // for __constant arguments. - // - // The associated constant memory space can be allocated in two - // ways. It can be explicitly initialized in the .ptx file where - // it is declared. Or, it can be allocated using the clCreateBuffer - // on the host. In this later case, the .ptx file will contain - // a global declaration of the parameter, but it will have an unknown - // array size. Thus, the symbol's address will not be set and we need - // to set it here before executing the PTX. - - char buffer[2048]; - snprintf(buffer, 2048, "%s_param_%u", m_name.c_str(), argn); - - symbol *p = m_symtab->lookup(buffer); - if (p == NULL) { - printf( - "GPGPU-Sim PTX: ERROR ** could not locate symbol for \'%s\' : cannot " - "bind buffer\n", - buffer); - abort(); - } - if (data) - p->set_address((addr_t) * (size_t *)data); - else { - // clSetKernelArg was passed NULL pointer for data... - // this is used for dynamically sized shared memory on NVIDIA platforms - bool is_ptr_shared = false; - if (i != m_ptx_kernel_param_info.end()) { - is_ptr_shared = i->second.is_ptr_shared(); + } + + // get reconvergence pc + reconvergence_pc = gpgpu_ctx->func_sim->get_converge_point(pc); + + m_decoded=true; +} + +void function_info::add_param_name_type_size( unsigned index, std::string name, int type, size_t size, bool ptr, memory_space_t space ) +{ + unsigned parsed_index; + char buffer[2048]; + snprintf(buffer,2048,"%s_param_%%u", m_name.c_str() ); + int ntokens = sscanf(name.c_str(),buffer,&parsed_index); + if( ntokens == 1 ) { + assert( m_ptx_kernel_param_info.find(parsed_index) == m_ptx_kernel_param_info.end() ); + m_ptx_kernel_param_info[parsed_index] = param_info(name, type, size, ptr, space); + } else { + assert( m_ptx_kernel_param_info.find(index) == m_ptx_kernel_param_info.end() ); + m_ptx_kernel_param_info[index] = param_info(name, type, size, ptr, space); + } +} + +void function_info::add_param_data( unsigned argn, struct gpgpu_ptx_sim_arg *args ) +{ + const void *data = args->m_start; + + bool scratchpad_memory_param = false; // Is this parameter in CUDA shared memory or OpenCL local memory + + std::map::iterator i=m_ptx_kernel_param_info.find(argn); + if( i != m_ptx_kernel_param_info.end() ) { + if (i->second.is_ptr_shared()) { + assert(args->m_start == NULL && "OpenCL parameter pointer to local memory must have NULL as value"); + scratchpad_memory_param = true; + } else { + param_t tmp; + tmp.pdata = args->m_start; + tmp.size = args->m_nbytes; + tmp.offset = args->m_offset; + tmp.type = 0; + i->second.add_data(tmp); + i->second.add_offset((unsigned) args->m_offset); } + } else { + scratchpad_memory_param = true; + } - if (!is_ptr_shared and !p->is_shared()) { - printf( - "GPGPU-Sim PTX: ERROR ** clSetKernelArg passed NULL but arg not " - "shared memory\n"); - abort(); + if (scratchpad_memory_param) { + // This should only happen for OpenCL: + // + // The LLVM PTX compiler in NVIDIA's driver (version 190.29) + // does not generate an argument in the function declaration + // for __constant arguments. + // + // The associated constant memory space can be allocated in two + // ways. It can be explicitly initialized in the .ptx file where + // it is declared. Or, it can be allocated using the clCreateBuffer + // on the host. In this later case, the .ptx file will contain + // a global declaration of the parameter, but it will have an unknown + // array size. Thus, the symbol's address will not be set and we need + // to set it here before executing the PTX. + + char buffer[2048]; + snprintf(buffer,2048,"%s_param_%u",m_name.c_str(),argn); + + symbol *p = m_symtab->lookup(buffer); + if( p == NULL ) { + printf("GPGPU-Sim PTX: ERROR ** could not locate symbol for \'%s\' : cannot bind buffer\n", buffer); + abort(); } - unsigned num_bits = 8 * args->m_nbytes; - printf( - "GPGPU-Sim PTX: deferred allocation of shared region for \"%s\" from " - "0x%x to 0x%x (shared memory space)\n", - p->name().c_str(), m_symtab->get_shared_next(), - m_symtab->get_shared_next() + num_bits / 8); - fflush(stdout); - assert((num_bits % 8) == 0); - addr_t addr = m_symtab->get_shared_next(); - addr_t addr_pad = - num_bits - ? (((num_bits / 8) - (addr % (num_bits / 8))) % (num_bits / 8)) - : 0; - p->set_address(addr + addr_pad); - m_symtab->alloc_shared(num_bits / 8 + addr_pad); - } - } + if( data ) + p->set_address((addr_t)*(size_t*)data); + else { + // clSetKernelArg was passed NULL pointer for data... + // this is used for dynamically sized shared memory on NVIDIA platforms + bool is_ptr_shared = false; + if( i != m_ptx_kernel_param_info.end() ) { + is_ptr_shared = i->second.is_ptr_shared(); + } + + if( !is_ptr_shared and !p->is_shared() ) { + printf("GPGPU-Sim PTX: ERROR ** clSetKernelArg passed NULL but arg not shared memory\n"); + abort(); + } + unsigned num_bits = 8*args->m_nbytes; + printf("GPGPU-Sim PTX: deferred allocation of shared region for \"%s\" from 0x%x to 0x%x (shared memory space)\n", + p->name().c_str(), + m_symtab->get_shared_next(), + m_symtab->get_shared_next() + num_bits/8 ); + fflush(stdout); + assert( (num_bits%8) == 0 ); + addr_t addr = m_symtab->get_shared_next(); + addr_t addr_pad = num_bits ? (((num_bits/8) - (addr % (num_bits/8))) % (num_bits/8)) : 0; + p->set_address( addr+addr_pad ); + m_symtab->alloc_shared( num_bits/8 + addr_pad ); + } + } } unsigned function_info::get_args_aligned_size() { - if (m_args_aligned_size >= 0) return m_args_aligned_size; - - unsigned param_address = 0; - unsigned int total_size = 0; - for (std::map::iterator i = - m_ptx_kernel_param_info.begin(); - i != m_ptx_kernel_param_info.end(); i++) { - param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); - - size_t arg_size = p.get_size() / 8; // size of param in bytes - total_size = (total_size + arg_size - 1) / arg_size * arg_size; // aligned - p.add_offset(total_size); - param->set_address(param_address + total_size); - total_size += arg_size; - } - - m_args_aligned_size = (total_size + 3) / 4 * 4; // final size aligned to word - - return m_args_aligned_size; -} - -void function_info::finalize(memory_space *param_mem) { - unsigned param_address = 0; - for (std::map::iterator i = - m_ptx_kernel_param_info.begin(); - i != m_ptx_kernel_param_info.end(); i++) { - param_info &p = i->second; - if (p.is_ptr_shared()) - continue; // Pointer to local memory: Should we pass the allocated shared - // memory address to the param memory space? - std::string name = p.get_name(); - int type = p.get_type(); - param_t param_value = p.get_value(); - param_value.type = type; - symbol *param = m_symtab->lookup(name.c_str()); - unsigned xtype = param->type()->get_key().scalar_type(); - assert(xtype == (unsigned)type); - size_t size; - size = param_value.size; // size of param in bytes - // assert(param_value.offset == param_address); - if (size != p.get_size() / 8) { - printf( - "GPGPU-Sim PTX: WARNING actual kernel paramter size = %zu bytes vs. " - "formal size = %zu (using smaller of two)\n", - size, p.get_size() / 8); - size = (size < (p.get_size() / 8)) ? size : (p.get_size() / 8); - } - // copy the parameter over word-by-word so that parameter that crosses a - // memory page can be copied over - // Jin: copy parameter using aligned rules - const type_info *paramtype = param->type(); - int align_amount = paramtype->get_key().get_alignment_spec(); - align_amount = (align_amount == -1) ? size : align_amount; - param_address = (param_address + align_amount - 1) / align_amount * - align_amount; // aligned - - const size_t word_size = 4; - // param_address = (param_address + size - 1) / size * size; //aligned with - // size - for (size_t idx = 0; idx < size; idx += word_size) { - const char *pdata = reinterpret_cast(param_value.pdata) + - idx; // cast to char * for ptr arithmetic - param_mem->write(param_address + idx, word_size, pdata, NULL, NULL); - } - unsigned offset = p.get_offset(); - assert(offset == param_address); - param->set_address(param_address); - param_address += size; - } -} - -void function_info::param_to_shared(memory_space *shared_mem, - symbol_table *symtab) { - // TODO: call this only for PTXPlus with GT200 models - // extern gpgpu_sim* g_the_gpu; - if (not gpgpu_ctx->the_gpgpusim->g_the_gpu->get_config().convert_to_ptxplus()) - return; - - // copies parameters into simulated shared memory - for (std::map::iterator i = - m_ptx_kernel_param_info.begin(); - i != m_ptx_kernel_param_info.end(); i++) { - param_info &p = i->second; - if (p.is_ptr_shared()) - continue; // Pointer to local memory: Should we pass the allocated shared - // memory address to the param memory space? - std::string name = p.get_name(); - int type = p.get_type(); - param_t value = p.get_value(); - value.type = type; - symbol *param = symtab->lookup(name.c_str()); - unsigned xtype = param->type()->get_key().scalar_type(); - assert(xtype == (unsigned)type); - - int tmp; - size_t size; - unsigned offset = p.get_offset(); - type_info_key::type_decode(xtype, size, tmp); - - // Write to shared memory - offset + 0x10 - shared_mem->write(offset + 0x10, size / 8, value.pdata, NULL, NULL); - } -} - -void function_info::list_param(FILE *fout) const { - for (std::map::const_iterator i = - m_ptx_kernel_param_info.begin(); - i != m_ptx_kernel_param_info.end(); i++) { - const param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); - addr_t param_addr = param->get_address(); - fprintf(fout, "%s: %#08x\n", name.c_str(), param_addr); - } - fflush(fout); -} - -void function_info::ptx_jit_config( - std::map mallocPtr_Size, - memory_space *param_mem, gpgpu_t *gpu, dim3 gridDim, dim3 blockDim) { - static unsigned long long counter = 0; - std::vector > param_data; - std::vector offsets; - std::vector paramIsPointer; - - char *gpgpusim_path = getenv("GPGPUSIM_ROOT"); - assert(gpgpusim_path != NULL); - char *wys_exec_path = getenv("WYS_EXEC_PATH"); - assert(wys_exec_path != NULL); - std::string command = - std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; - std::string filename(std::string(gpgpusim_path) + - "/debug_tools/WatchYourStep/data/params.config" + - std::to_string(counter)); - - // initialize paramList - char buff[1024]; - std::string filename_c(filename + "_c"); - snprintf(buff, 1024, "c++filt %s > %s", get_name().c_str(), - filename_c.c_str()); - assert(system(buff) != NULL); - FILE *fp = fopen(filename_c.c_str(), "r"); - fgets(buff, 1024, fp); - fclose(fp); - std::string fn(buff); - size_t pos1, pos2; - pos1 = fn.find_last_of("("); - pos2 = fn.find(")", pos1); - assert(pos2 > pos1 && pos1 > 0); - strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); - char *tok; - tok = strtok(buff, ","); - std::string tmp; - while (tok != NULL) { - std::string param(tok); - if (param.find("<") != std::string::npos) { - assert(param.find(">") == std::string::npos); - assert(param.find("*") == std::string::npos); - tmp = param; - } else { - if (tmp.length() > 0) { - tmp = ""; - assert(param.find(">") != std::string::npos); - assert(param.find("<") == std::string::npos); - assert(param.find("*") == std::string::npos); - } - printf("%s\n", param.c_str()); - if (param.find("*") != std::string::npos) { - paramIsPointer.push_back(true); - } else { - paramIsPointer.push_back(false); + + if(m_args_aligned_size >= 0) + return m_args_aligned_size; + + unsigned param_address = 0; + unsigned int total_size = 0; + for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + + size_t arg_size = p.get_size() / 8; // size of param in bytes + total_size = (total_size + arg_size - 1) / arg_size * arg_size; //aligned + p.add_offset(total_size); + param->set_address(param_address + total_size); + total_size += arg_size; + } + + m_args_aligned_size = (total_size + 3) / 4 * 4; //final size aligned to word + + return m_args_aligned_size; + +} + + +void function_info::finalize( memory_space *param_mem ) +{ + unsigned param_address = 0; + for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + param_info &p = i->second; + if (p.is_ptr_shared()) continue; // Pointer to local memory: Should we pass the allocated shared memory address to the param memory space? + std::string name = p.get_name(); + int type = p.get_type(); + param_t param_value = p.get_value(); + param_value.type = type; + symbol *param = m_symtab->lookup(name.c_str()); + unsigned xtype = param->type()->get_key().scalar_type(); + assert(xtype==(unsigned)type); + size_t size; + size = param_value.size; // size of param in bytes + // assert(param_value.offset == param_address); + if( size != p.get_size() / 8) { + printf("GPGPU-Sim PTX: WARNING actual kernel paramter size = %zu bytes vs. formal size = %zu (using smaller of two)\n", + size, p.get_size()/8); + size = (size<(p.get_size()/8))?size:(p.get_size()/8); + } + // copy the parameter over word-by-word so that parameter that crosses a memory page can be copied over + //Jin: copy parameter using aligned rules + const type_info *paramtype = param->type(); + int align_amount = paramtype->get_key().get_alignment_spec(); + align_amount = (align_amount == -1) ? size : align_amount; + param_address = (param_address + align_amount - 1) / align_amount * align_amount; //aligned + + const size_t word_size = 4; + //param_address = (param_address + size - 1) / size * size; //aligned with size + for (size_t idx = 0; idx < size; idx += word_size) { + const char *pdata = reinterpret_cast(param_value.pdata) + idx; // cast to char * for ptr arithmetic + param_mem->write(param_address + idx, word_size, pdata,NULL,NULL); } + unsigned offset = p.get_offset(); + assert(offset == param_address); + param->set_address(param_address); + param_address += size; + } +} + +void function_info::param_to_shared( memory_space *shared_mem, symbol_table *symtab ) +{ + // TODO: call this only for PTXPlus with GT200 models + //extern gpgpu_sim* g_the_gpu; + if (not gpgpu_ctx->the_gpgpusim->g_the_gpu->get_config().convert_to_ptxplus()) return; + + // copies parameters into simulated shared memory + for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + param_info &p = i->second; + if (p.is_ptr_shared()) continue; // Pointer to local memory: Should we pass the allocated shared memory address to the param memory space? + std::string name = p.get_name(); + int type = p.get_type(); + param_t value = p.get_value(); + value.type = type; + symbol *param = symtab->lookup(name.c_str()); + unsigned xtype = param->type()->get_key().scalar_type(); + assert(xtype==(unsigned)type); + + int tmp; + size_t size; + unsigned offset = p.get_offset(); + type_info_key::type_decode(xtype,size,tmp); + + // Write to shared memory - offset + 0x10 + shared_mem->write(offset+0x10,size/8,value.pdata,NULL,NULL); + } +} + + +void function_info::list_param( FILE *fout ) const +{ + for( std::map::const_iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + const param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); + fprintf(fout, "%s: %#08x\n", name.c_str(), param_addr); + } + fflush(fout); +} + +void function_info::ptx_jit_config(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) +{ + static unsigned long long counter = 0; + std::vector< std::pair > param_data; + std::vector offsets; + std::vector paramIsPointer; + + char * gpgpusim_path = getenv("GPGPUSIM_ROOT"); + assert(gpgpusim_path!=NULL); + char * wys_exec_path = getenv("WYS_EXEC_PATH"); + assert(wys_exec_path!=NULL); + std::string command = std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; + std::string filename(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/params.config" + std::to_string(counter)); + + //initialize paramList + char buff[1024]; + std::string filename_c(filename+"_c"); + snprintf(buff,1024,"c++filt %s > %s", get_name().c_str(), filename_c.c_str()); + assert(system(buff) != NULL); + FILE *fp = fopen(filename_c.c_str(), "r"); + fgets(buff, 1024, fp); + fclose(fp); + std::string fn(buff); + size_t pos1, pos2; + pos1 = fn.find_last_of("("); + pos2 = fn.find(")", pos1); + assert(pos2>pos1&&pos1>0); + strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); + char *tok; + tok = strtok(buff, ","); + std::string tmp; + while(tok!=NULL){ + std::string param(tok); + if(param.find("<")!=std::string::npos){ + assert(param.find(">")==std::string::npos); + assert(param.find("*")==std::string::npos); + tmp = param; + } else { + if (tmp.length()>0){ + tmp = ""; + assert(param.find(">")!=std::string::npos); + assert(param.find("<")==std::string::npos); + assert(param.find("*")==std::string::npos); + } + printf("%s\n", param.c_str()); + if(param.find("*")!=std::string::npos){ + paramIsPointer.push_back(true); + }else{ + paramIsPointer.push_back(false); + } + } + tok = strtok(NULL, ","); } - tok = strtok(NULL, ","); - } - - for (std::map::iterator i = - m_ptx_kernel_param_info.begin(); - i != m_ptx_kernel_param_info.end(); i++) { - param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); - addr_t param_addr = param->get_address(); - param_t param_value = p.get_value(); - offsets.push_back((unsigned)p.get_offset()); - - if (paramIsPointer[i->first] && - (*(unsigned long long *)param_value.pdata != 0)) { - // is pointer - assert(param_value.size == sizeof(void *) && - "MisID'd this param as pointer"); - size_t array_size = 0; - unsigned long long param_pointer = - *(unsigned long long *)param_value.pdata; - if (mallocPtr_Size.find(param_pointer) != mallocPtr_Size.end()) { - array_size = mallocPtr_Size[param_pointer]; - } else { - for (std::map::iterator j = - mallocPtr_Size.begin(); - j != mallocPtr_Size.end(); j++) { - if (param_pointer > j->first && - param_pointer < j->first + j->second) { - array_size = j->first + j->second - param_pointer; - break; - } + + + for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); + param_t param_value = p.get_value(); + offsets.push_back((unsigned)p.get_offset()); + + if (paramIsPointer[i->first] && (*(unsigned long long*)param_value.pdata != 0)){ + //is pointer + assert(param_value.size==sizeof(void*)&&"MisID'd this param as pointer"); + size_t array_size = 0; + unsigned long long param_pointer = *(unsigned long long*)param_value.pdata; + if(mallocPtr_Size.find(param_pointer)!=mallocPtr_Size.end()){ + array_size = mallocPtr_Size[param_pointer]; + }else{ + for( std::map::iterator j=mallocPtr_Size.begin(); j!=mallocPtr_Size.end(); j++ ) { + if(param_pointer>j->first&¶m_pointerfirst + j->second){ + array_size = j->first + j->second - param_pointer; + break; + } + } + assert(array_size>0&&"pointer was not previously malloc'd"); + } + + unsigned char* val = (unsigned char*) malloc(param_value.size); + param_mem->read(param_addr,param_value.size,(void*)val); + unsigned char* array_val = (unsigned char*) malloc(array_size); + gpu->get_global_memory()->read(*(unsigned*)((void*)val),array_size,(void*)array_val); + param_data.push_back(std::pair(array_size,array_val)); + paramIsPointer.push_back(true); + }else{ + unsigned char* val = (unsigned char*) malloc(param_value.size); + param_mem->read(param_addr,param_value.size,(void*)val); + param_data.push_back(std::pair(param_value.size,val)); + paramIsPointer.push_back(false); } - assert(array_size > 0 && "pointer was not previously malloc'd"); - } + } - unsigned char *val = (unsigned char *)malloc(param_value.size); - param_mem->read(param_addr, param_value.size, (void *)val); - unsigned char *array_val = (unsigned char *)malloc(array_size); - gpu->get_global_memory()->read(*(unsigned *)((void *)val), array_size, - (void *)array_val); - param_data.push_back( - std::pair(array_size, array_val)); - paramIsPointer.push_back(true); - } else { - unsigned char *val = (unsigned char *)malloc(param_value.size); - param_mem->read(param_addr, param_value.size, (void *)val); - param_data.push_back( - std::pair(param_value.size, val)); - paramIsPointer.push_back(false); + FILE *fout = fopen (filename.c_str(), "w"); + printf("Writing data to %s ...\n", filename.c_str()); + fprintf(fout, "%s\n", get_name().c_str()); + fprintf(fout, "%u,%u,%u %u,%u,%u\n", gridDim.x, gridDim.y, gridDim.z, blockDim.x, blockDim.y, blockDim.z); + size_t index = 0; + for( std::vector< std::pair >::const_iterator i=param_data.begin(); i!=param_data.end(); i++ ) { + if (paramIsPointer[index]){ + fprintf(fout, "*"); + } + fprintf(fout, "%lu :", i->first); + for (size_t j = 0; jfirst; j++){ + fprintf(fout, " %u", i->second[j]); + } + fprintf(fout, " : %u", offsets[index]); + free (i->second); + fprintf(fout, "\n"); + index++; } - } - - FILE *fout = fopen(filename.c_str(), "w"); - printf("Writing data to %s ...\n", filename.c_str()); - fprintf(fout, "%s\n", get_name().c_str()); - fprintf(fout, "%u,%u,%u %u,%u,%u\n", gridDim.x, gridDim.y, gridDim.z, - blockDim.x, blockDim.y, blockDim.z); - size_t index = 0; - for (std::vector >::const_iterator i = - param_data.begin(); - i != param_data.end(); i++) { - if (paramIsPointer[index]) { - fprintf(fout, "*"); + fflush(fout); + fclose(fout); + + //ptx config + std::string ptx_config_fn(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/ptx.config" + std::to_string(counter)); + snprintf(buff, 1024, "grep -rn \".entry %s\" %s/*.ptx | cut -d \":\" -f 1-2 > %s", get_name().c_str(), wys_exec_path, ptx_config_fn.c_str()); + if (system(buff)!=0){ + printf("WARNING: Failed to execute grep to find ptx source \n"); + printf("Problematic call: %s", buff); + abort(); } - fprintf(fout, "%lu :", i->first); - for (size_t j = 0; j < i->first; j++) { - fprintf(fout, " %u", i->second[j]); + FILE *fin = fopen(ptx_config_fn.c_str(), "r"); + char ptx_source[256]; + unsigned line_number; + int numscanned = fscanf(fin, "%[^:]:%u", ptx_source, &line_number); + assert(numscanned == 2); + fclose(fin); + snprintf(buff, 1024, "grep -rn \".version\" %s | cut -d \":\" -f 1 | xargs -I \"{}\" awk \"NR>={}&&NR<={}+2\" %s > %s", ptx_source, ptx_source, ptx_config_fn.c_str()); + if (system(buff)!=0){ + printf("WARNING: Failed to execute grep to find ptx header \n"); + printf("Problematic call: %s", buff); + abort(); } - fprintf(fout, " : %u", offsets[index]); - free(i->second); - fprintf(fout, "\n"); - index++; - } - fflush(fout); - fclose(fout); - - // ptx config - std::string ptx_config_fn(std::string(gpgpusim_path) + - "/debug_tools/WatchYourStep/data/ptx.config" + - std::to_string(counter)); - snprintf(buff, 1024, - "grep -rn \".entry %s\" %s/*.ptx | cut -d \":\" -f 1-2 > %s", - get_name().c_str(), wys_exec_path, ptx_config_fn.c_str()); - if (system(buff) != 0) { - printf("WARNING: Failed to execute grep to find ptx source \n"); - printf("Problematic call: %s", buff); - abort(); - } - FILE *fin = fopen(ptx_config_fn.c_str(), "r"); - char ptx_source[256]; - unsigned line_number; - int numscanned = fscanf(fin, "%[^:]:%u", ptx_source, &line_number); - assert(numscanned == 2); - fclose(fin); - snprintf(buff, 1024, - "grep -rn \".version\" %s | cut -d \":\" -f 1 | xargs -I \"{}\" awk " - "\"NR>={}&&NR<={}+2\" %s > %s", - ptx_source, ptx_source, ptx_config_fn.c_str()); - if (system(buff) != 0) { - printf("WARNING: Failed to execute grep to find ptx header \n"); - printf("Problematic call: %s", buff); - abort(); - } - fin = fopen(ptx_source, "r"); - assert(fin != NULL); - printf("Writing data to %s ...\n", ptx_config_fn.c_str()); - fout = fopen(ptx_config_fn.c_str(), "a"); - assert(fout != NULL); - for (unsigned i = 0; i < line_number; i++) { - assert(fgets(buff, 1024, fin) != NULL); - assert(!feof(fin)); - } - fprintf(fout, "\n\n"); - do { - fprintf(fout, "%s", buff); - assert(fgets(buff, 1024, fin) != NULL); - if (feof(fin)) { - break; + fin = fopen(ptx_source, "r"); + assert(fin!=NULL); + printf("Writing data to %s ...\n", ptx_config_fn.c_str()); + fout = fopen(ptx_config_fn.c_str(), "a"); + assert(fout!=NULL); + for (unsigned i = 0; i -bool cuda_sim::ptx_debug_exec_dump_cond(int thd_uid, addr_t pc) { - if (g_debug_execution >= activate_level) { - // check each type of debug dump constraint to filter out dumps - if ((g_debug_thread_uid != 0) && - (thd_uid != (unsigned)g_debug_thread_uid)) { - return false; - } - if ((g_debug_pc != 0xBEEF1518) && (pc != g_debug_pc)) { - return false; - } +template +bool cuda_sim::ptx_debug_exec_dump_cond(int thd_uid, addr_t pc) +{ + if (g_debug_execution >= activate_level) { + // check each type of debug dump constraint to filter out dumps + if ( (g_debug_thread_uid != 0) && (thd_uid != (unsigned)g_debug_thread_uid) ) { + return false; + } + if ( (g_debug_pc != 0xBEEF1518) && (pc != g_debug_pc) ) { + return false; + } - return true; - } - - return false; -} - -void cuda_sim::init_inst_classification_stat() { - static std::set init; - if (init.find(g_ptx_kernel_count) != init.end()) return; - init.insert(g_ptx_kernel_count); - -#define MAX_CLASS_KER 1024 - char kernelname[MAX_CLASS_KER] = ""; - if (!g_inst_classification_stat) - g_inst_classification_stat = (void **)calloc(MAX_CLASS_KER, sizeof(void *)); - snprintf(kernelname, MAX_CLASS_KER, "Kernel %d Classification\n", - g_ptx_kernel_count); - assert(g_ptx_kernel_count < MAX_CLASS_KER); // a static limit on number of - // kernels increase it if it - // fails! - g_inst_classification_stat[g_ptx_kernel_count] = - StatCreate(kernelname, 1, 20); - if (!g_inst_op_classification_stat) - g_inst_op_classification_stat = - (void **)calloc(MAX_CLASS_KER, sizeof(void *)); - snprintf(kernelname, MAX_CLASS_KER, "Kernel %d OP Classification\n", - g_ptx_kernel_count); - g_inst_op_classification_stat[g_ptx_kernel_count] = - StatCreate(kernelname, 1, 100); -} - -static unsigned get_tex_datasize(const ptx_instruction *pI, - ptx_thread_info *thread) { - const operand_info &src1 = pI->src1(); // the name of the texture - std::string texname = src1.name(); - - /* - For programs with many streams, textures can be bound and unbound - asynchronously. This means we need to use the kernel's "snapshot" of - the state of the texture mappings when it was launched (so that we - don't try to access the incorrect texture mapping if it's been updated, - or that we don't access a mapping that has been unbound). - */ - kernel_info_t &k = thread->get_kernel(); - const struct textureInfo *texInfo = k.get_texinfo(texname); - - unsigned data_size = texInfo->texel_size; - return data_size; -} - -int tensorcore_op(int inst_opcode) { - if ((inst_opcode == MMA_OP) || (inst_opcode == MMA_LD_OP) || - (inst_opcode == MMA_ST_OP)) - return 1; - else - return 0; -} -void ptx_thread_info::ptx_exec_inst(warp_inst_t &inst, unsigned lane_id) { - bool skip = false; - int op_classification = 0; - addr_t pc = next_instr(); - assert(pc == - inst.pc); // make sure timing model and functional model are in sync - const ptx_instruction *pI = m_func_info->get_instruction(pc); - - set_npc(pc + pI->inst_size()); - - try { - clearRPC(); - m_last_set_operand_value.u64 = 0; - - if (is_done()) { - printf( - "attempted to execute instruction on a thread that is already " - "done.\n"); + return true; + } + + return false; +} + +void cuda_sim::init_inst_classification_stat() +{ + static std::set init; + if( init.find(g_ptx_kernel_count) != init.end() ) + return; + init.insert(g_ptx_kernel_count); + + #define MAX_CLASS_KER 1024 + char kernelname[MAX_CLASS_KER] =""; + if (!g_inst_classification_stat) g_inst_classification_stat = (void**)calloc(MAX_CLASS_KER, sizeof(void*)); + snprintf(kernelname, MAX_CLASS_KER, "Kernel %d Classification\n",g_ptx_kernel_count ); + assert( g_ptx_kernel_count < MAX_CLASS_KER ) ; // a static limit on number of kernels increase it if it fails! + g_inst_classification_stat[g_ptx_kernel_count] = StatCreate(kernelname,1,20); + if (!g_inst_op_classification_stat) g_inst_op_classification_stat = (void**)calloc(MAX_CLASS_KER, sizeof(void*)); + snprintf(kernelname, MAX_CLASS_KER, "Kernel %d OP Classification\n",g_ptx_kernel_count ); + g_inst_op_classification_stat[g_ptx_kernel_count] = StatCreate(kernelname,1,100); +} + +static unsigned get_tex_datasize( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &src1 = pI->src1(); //the name of the texture + std::string texname = src1.name(); + + /* + For programs with many streams, textures can be bound and unbound + asynchronously. This means we need to use the kernel's "snapshot" of + the state of the texture mappings when it was launched (so that we + don't try to access the incorrect texture mapping if it's been updated, + or that we don't access a mapping that has been unbound). + */ + kernel_info_t& k = thread->get_kernel(); + const struct textureInfo* texInfo = k.get_texinfo(texname); + + unsigned data_size = texInfo->texel_size; + return data_size; +} + +int tensorcore_op(int inst_opcode){ + + if((inst_opcode==MMA_OP)||(inst_opcode==MMA_LD_OP)||(inst_opcode==MMA_ST_OP)) + return 1; + else + return 0; +} +void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id) +{ + + bool skip = false; + int op_classification = 0; + addr_t pc = next_instr(); + assert( pc == inst.pc ); // make sure timing model and functional model are in sync + const ptx_instruction *pI = m_func_info->get_instruction(pc); + + set_npc( pc + pI->inst_size() ); + + + try { + + clearRPC(); + m_last_set_operand_value.u64 = 0; + + if(is_done()) + { + printf("attempted to execute instruction on a thread that is already done.\n"); assert(0); - } - - if (g_debug_execution >= 6 || - m_gpu->get_config().get_ptx_inst_debug_to_file()) { - if ((m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid == 0) || - (get_uid() == - (unsigned)(m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid))) { - clear_modifiedregs(); - enable_debug_trace(); + } + + if ( g_debug_execution >= 6 || m_gpu->get_config().get_ptx_inst_debug_to_file()) { + if ( (m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid==0) + || (get_uid() == (unsigned)(m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid)) ) { + + clear_modifiedregs(); + enable_debug_trace(); } - } - - if (pI->has_pred()) { + } + + + if( pI->has_pred() ) { const operand_info &pred = pI->get_pred(); ptx_reg_t pred_value = get_operand_value(pred, pred, PRED_TYPE, this, 0); - if (pI->get_pred_mod() == -1) { - skip = (pred_value.pred & 0x0001) ^ - pI->get_pred_neg(); // ptxplus inverts the zero flag + if(pI->get_pred_mod() == -1) { + skip = (pred_value.pred & 0x0001) ^ pI->get_pred_neg(); //ptxplus inverts the zero flag } else { - skip = !pred_lookup(pI->get_pred_mod(), pred_value.pred & 0x000F); + skip = !pred_lookup(pI->get_pred_mod(), pred_value.pred & 0x000F); } - } - int inst_opcode = pI->get_opcode(); - - if (skip) { + } + int inst_opcode=pI->get_opcode(); + + if( skip ) { inst.set_not_active(lane_id); - } else { + } else { const ptx_instruction *pI_saved = pI; ptx_instruction *pJ = NULL; - if (pI->get_opcode() == VOTE_OP) { - pJ = new ptx_instruction(*pI); - *((warp_inst_t *)pJ) = inst; // copy active mask information - pI = pJ; + if( pI->get_opcode() == VOTE_OP ) { + pJ = new ptx_instruction(*pI); + *((warp_inst_t*)pJ) = inst; // copy active mask information + pI = pJ; } - - if (((inst_opcode == MMA_OP || inst_opcode == MMA_LD_OP || - inst_opcode == MMA_ST_OP))) { - if (inst.active_count() != MAX_WARP_SIZE) { - printf( - "Tensor Core operation are warp synchronous operation. All the " - "threads needs to be active."); - assert(0); - } + + if(((inst_opcode==MMA_OP||inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))){ + if(inst.active_count()!=MAX_WARP_SIZE) + { + printf("Tensor Core operation are warp synchronous operation. All the threads needs to be active."); + assert(0); + } } - - // Tensorcore is warp synchronous operation. So these instructions needs - // to be executed only once. To make the simulation faster removing the - // redundant tensorcore operation - if (!tensorcore_op(inst_opcode) || - ((tensorcore_op(inst_opcode)) && (lane_id == 0))) { - switch (inst_opcode) { -#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ - case OP: \ - FUNC(pI, this); \ - op_classification = CLASSIFICATION; \ - break; -#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ - case OP: \ - FUNC(pI, get_core(), inst); \ - op_classification = CLASSIFICATION; \ - break; -#include "opcodes.def" -#undef OP_DEF -#undef OP_W_DEF - default: - printf("Execution error: Invalid opcode (0x%x)\n", - pI->get_opcode()); - break; - } + + //Tensorcore is warp synchronous operation. So these instructions needs to be executed only once. To make the simulation faster removing the redundant tensorcore operation + if(!tensorcore_op(inst_opcode)||((tensorcore_op(inst_opcode))&&(lane_id==0))){ + switch ( inst_opcode ) { + #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,this); op_classification = CLASSIFICATION; break; + #define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,get_core(),inst); op_classification = CLASSIFICATION; break; + #include "opcodes.def" + #undef OP_DEF + #undef OP_W_DEF + default: printf( "Execution error: Invalid opcode (0x%x)\n", pI->get_opcode() ); break; + } } delete pJ; pI = pI_saved; - + // Run exit instruction if exit option included - if (pI->is_exit()) exit_impl(pI, this); - } - - const gpgpu_functional_sim_config &config = m_gpu->get_config(); - - // Output instruction information to file and stdout - if (config.get_ptx_inst_debug_to_file() != 0 && - (config.get_ptx_inst_debug_thread_uid() == 0 || - config.get_ptx_inst_debug_thread_uid() == get_uid())) { - fprintf(m_gpu->get_ptx_inst_debug_file(), "[thd=%u] : (%s:%u - %s)\n", - get_uid(), pI->source_file(), pI->source_line(), - pI->get_source()); - // fprintf(ptx_inst_debug_file, "has memory read=%d, has memory - // write=%d\n", pI->has_memory_read(), pI->has_memory_write()); + if(pI->is_exit()) + exit_impl(pI,this); + } + + + + const gpgpu_functional_sim_config &config = m_gpu->get_config(); + + // Output instruction information to file and stdout + if( config.get_ptx_inst_debug_to_file() != 0 && + (config.get_ptx_inst_debug_thread_uid() == 0 || config.get_ptx_inst_debug_thread_uid() == get_uid()) ) { + fprintf(m_gpu->get_ptx_inst_debug_file(), + "[thd=%u] : (%s:%u - %s)\n", + get_uid(), + pI->source_file(), pI->source_line(), pI->get_source() ); + //fprintf(ptx_inst_debug_file, "has memory read=%d, has memory write=%d\n", pI->has_memory_read(), pI->has_memory_write()); fflush(m_gpu->get_ptx_inst_debug_file()); - } + } - if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<5>(get_uid(), - pc)) { + if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<5>(get_uid(), pc) ) { dim3 ctaid = get_ctaid(); dim3 tid = get_tid(); - printf( - "%u [thd=%u][i=%u] : ctaid=(%u,%u,%u) tid=(%u,%u,%u) icount=%u " - "[pc=%u] (%s:%u - %s) [0x%llx]\n", - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, get_uid(), pI->uid(), - ctaid.x, ctaid.y, ctaid.z, tid.x, tid.y, tid.z, get_icount(), pc, - pI->source_file(), pI->source_line(), pI->get_source(), - m_last_set_operand_value.u64); + printf("%u [thd=%u][i=%u] : ctaid=(%u,%u,%u) tid=(%u,%u,%u) icount=%u [pc=%u] (%s:%u - %s) [0x%llx]\n", + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, + get_uid(), + pI->uid(), ctaid.x,ctaid.y,ctaid.z,tid.x,tid.y,tid.z, + get_icount(), + pc, pI->source_file(), pI->source_line(), pI->get_source(), + m_last_set_operand_value.u64 ); fflush(stdout); - } - - addr_t insn_memaddr = 0xFEEBDAED; - memory_space_t insn_space = undefined_space; - _memory_op_t insn_memory_op = no_memory_op; - unsigned insn_data_size = 0; - if ((pI->has_memory_read() || pI->has_memory_write())) { - if (!((inst_opcode == MMA_LD_OP || inst_opcode == MMA_ST_OP))) { + } + + addr_t insn_memaddr = 0xFEEBDAED; + memory_space_t insn_space = undefined_space; + _memory_op_t insn_memory_op = no_memory_op; + unsigned insn_data_size = 0; + if ( (pI->has_memory_read() || pI->has_memory_write()) ) { + if(!((inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))) + { insn_memaddr = last_eaddr(); insn_space = last_space(); unsigned to_type = pI->get_type(); insn_data_size = datatype2size(to_type); insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; - } - } - - if (pI->get_opcode() == BAR_OP && pI->barrier_op() == RED_OPTION) { - inst.add_callback(lane_id, last_callback().function, - last_callback().instruction, this, - false /*not atomic*/); - } + } + } + + if ( pI->get_opcode() == BAR_OP && pI->barrier_op() == RED_OPTION) { + inst.add_callback( lane_id, last_callback().function, last_callback().instruction, this,false /*not atomic*/); + } - if (pI->get_opcode() == ATOM_OP) { + if ( pI->get_opcode() == ATOM_OP ) { insn_memaddr = last_eaddr(); insn_space = last_space(); - inst.add_callback(lane_id, last_callback().function, - last_callback().instruction, this, true /*atomic*/); + inst.add_callback( lane_id, last_callback().function, last_callback().instruction, this,true /*atomic*/); unsigned to_type = pI->get_type(); insn_data_size = datatype2size(to_type); - } + } - if (pI->get_opcode() == TEX_OP) { - inst.set_addr(lane_id, last_eaddr()); - assert(inst.space == last_space()); - insn_data_size = get_tex_datasize( - pI, - this); // texture obtain its data granularity from the texture info - } + if (pI->get_opcode() == TEX_OP) { + inst.set_addr(lane_id, last_eaddr() ); + assert( inst.space == last_space() ); + insn_data_size = get_tex_datasize(pI, this); // texture obtain its data granularity from the texture info + } - // Output register information to file and stdout - if (config.get_ptx_inst_debug_to_file() != 0 && - (config.get_ptx_inst_debug_thread_uid() == 0 || - config.get_ptx_inst_debug_thread_uid() == get_uid())) { + // Output register information to file and stdout + if( config.get_ptx_inst_debug_to_file()!=0 && + (config.get_ptx_inst_debug_thread_uid()==0||config.get_ptx_inst_debug_thread_uid()==get_uid()) ) { dump_modifiedregs(m_gpu->get_ptx_inst_debug_file()); dump_regs(m_gpu->get_ptx_inst_debug_file()); - } - - if (g_debug_execution >= 6) { - if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<6>(get_uid(), - pc)) - dump_modifiedregs(stdout); - } - if (g_debug_execution >= 10) { - if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<10>(get_uid(), - pc)) - dump_regs(stdout); - } - update_pc(); - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn++; - - // not using it with functional simulation mode - if (!(this->m_functionalSimulationMode)) - ptx_file_line_stats_add_exec_count(pI); + } - if (m_gpu->gpgpu_ctx->func_sim->gpgpu_ptx_instruction_classification) { + if ( g_debug_execution >= 6 ) { + if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<6>(get_uid(), pc) ) + dump_modifiedregs(stdout); + } + if ( g_debug_execution >= 10 ) { + if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<10>(get_uid(), pc) ) + dump_regs(stdout); + } + update_pc(); + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn++; + + //not using it with functional simulation mode + if(!(this->m_functionalSimulationMode)) + ptx_file_line_stats_add_exec_count(pI); + + if ( m_gpu->gpgpu_ctx->func_sim->gpgpu_ptx_instruction_classification ) { m_gpu->gpgpu_ctx->func_sim->init_inst_classification_stat(); - unsigned space_type = 0; - switch (pI->get_space().get_type()) { - case global_space: - space_type = 10; - break; - case local_space: - space_type = 11; - break; - case tex_space: - space_type = 12; - break; - case surf_space: - space_type = 13; - break; - case param_space_kernel: - case param_space_local: - space_type = 14; - break; - case shared_space: - space_type = 15; - break; - case const_space: - space_type = 16; - break; - default: - space_type = 0; - break; + unsigned space_type=0; + switch ( pI->get_space().get_type() ) { + case global_space: space_type = 10; break; + case local_space: space_type = 11; break; + case tex_space: space_type = 12; break; + case surf_space: space_type = 13; break; + case param_space_kernel: + case param_space_local: + space_type = 14; break; + case shared_space: space_type = 15; break; + case const_space: space_type = 16; break; + default: + space_type = 0 ; + break; } - StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat - [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], - op_classification); - if (space_type) - StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat - [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], - (int)space_type); - StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_op_classification_stat - [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], - (int)pI->get_opcode()); - } - if ((m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn % 100000) == 0) { + StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], op_classification); + if (space_type) StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], ( int )space_type); + StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_op_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], (int) pI->get_opcode() ); + } + if ( (m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn % 100000) == 0 ) { dim3 ctaid = get_ctaid(); dim3 tid = get_tid(); - DPRINTF(LIVENESS, - "GPGPU-Sim PTX: %u instructions simulated : ctaid=(%u,%u,%u) " - "tid=(%u,%u,%u)\n", - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, ctaid.x, ctaid.y, - ctaid.z, tid.x, tid.y, tid.z); + DPRINTF(LIVENESS, "GPGPU-Sim PTX: %u instructions simulated : ctaid=(%u,%u,%u) tid=(%u,%u,%u)\n", + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, ctaid.x,ctaid.y,ctaid.z,tid.x,tid.y,tid.z ); fflush(stdout); - } - - // "Return values" - if (!skip) { - if (!((inst_opcode == MMA_LD_OP || inst_opcode == MMA_ST_OP))) { - inst.space = insn_space; - inst.set_addr(lane_id, insn_memaddr); - inst.data_size = insn_data_size; // simpleAtomicIntrinsics - assert(inst.memory_op == insn_memory_op); + } + + // "Return values" + if(!skip) { + if(!((inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))) + { + inst.space = insn_space; + inst.set_addr(lane_id, insn_memaddr); + inst.data_size = insn_data_size; // simpleAtomicIntrinsics + assert( inst.memory_op == insn_memory_op ); + } + } + + } catch ( int x ) { + printf("GPGPU-Sim PTX: ERROR (%d) executing intruction (%s:%u)\n", x, pI->source_file(), pI->source_line() ); + printf("GPGPU-Sim PTX: '%s'\n", pI->get_source() ); + abort(); + } + +} + +void cuda_sim::set_param_gpgpu_num_shaders(int num_shaders) +{ + gpgpu_param_num_shaders = num_shaders; +} + +const struct gpgpu_ptx_sim_info* ptx_sim_kernel_info(const function_info *kernel) +{ + return kernel->get_kernel_info(); +} + +const warp_inst_t *gpgpu_context::ptx_fetch_inst( address_type pc ) +{ + return pc_to_instruction(pc); +} + +unsigned ptx_sim_init_thread( kernel_info_t &kernel, + ptx_thread_info** thread_info, + int sid, + unsigned tid, + unsigned threads_left, + unsigned num_threads, + core_t *core, + unsigned hw_cta_id, + unsigned hw_warp_id, + gpgpu_t *gpu, + bool isInFunctionalSimulationMode) +{ + std::list &active_threads = kernel.active_threads(); + + static std::map shared_memory_lookup; + static std::map sstarr_memory_lookup; + static std::map ptx_cta_lookup; + static std::map ptx_warp_lookup; + static std::map > local_memory_lookup; + + if ( *thread_info != NULL ) { + ptx_thread_info *thd = *thread_info; + assert( thd->is_done() ); + if ( g_debug_execution==-1 ) { + dim3 ctaid = thd->get_ctaid(); + dim3 t = thd->get_tid(); + printf("GPGPU-Sim PTX simulator: thread exiting ctaid=(%u,%u,%u) tid=(%u,%u,%u) uid=%u\n", + ctaid.x,ctaid.y,ctaid.z,t.x,t.y,t.z, thd->get_uid() ); + fflush(stdout); } - } + thd->m_cta_info->register_deleted_thread(thd); + delete thd; + *thread_info = NULL; + } - } catch (int x) { - printf("GPGPU-Sim PTX: ERROR (%d) executing intruction (%s:%u)\n", x, - pI->source_file(), pI->source_line()); - printf("GPGPU-Sim PTX: '%s'\n", pI->get_source()); - abort(); - } -} + if ( !active_threads.empty() ) { + assert( active_threads.size() <= threads_left ); + ptx_thread_info *thd = active_threads.front(); + active_threads.pop_front(); + *thread_info = thd; + thd->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, isInFunctionalSimulationMode ); + return 1; + } -void cuda_sim::set_param_gpgpu_num_shaders(int num_shaders) { - gpgpu_param_num_shaders = num_shaders; -} + if ( kernel.no_more_ctas_to_run() ) { + return 0; //finished! + } -const struct gpgpu_ptx_sim_info *ptx_sim_kernel_info( - const function_info *kernel) { - return kernel->get_kernel_info(); -} + if ( threads_left < kernel.threads_per_cta() ) { + return 0; + } -const warp_inst_t *gpgpu_context::ptx_fetch_inst(address_type pc) { - return pc_to_instruction(pc); -} + if ( g_debug_execution==-1 ) { + printf("GPGPU-Sim PTX simulator: STARTING THREAD ALLOCATION --> \n"); + fflush(stdout); + } -unsigned ptx_sim_init_thread(kernel_info_t &kernel, - ptx_thread_info **thread_info, int sid, - unsigned tid, unsigned threads_left, - unsigned num_threads, core_t *core, - unsigned hw_cta_id, unsigned hw_warp_id, - gpgpu_t *gpu, bool isInFunctionalSimulationMode) { - std::list &active_threads = kernel.active_threads(); + //initializing new CTA + ptx_cta_info *cta_info = NULL; + memory_space *shared_mem = NULL; + memory_space *sstarr_mem = NULL; - static std::map shared_memory_lookup; - static std::map sstarr_memory_lookup; - static std::map ptx_cta_lookup; - static std::map ptx_warp_lookup; - static std::map > - local_memory_lookup; + unsigned cta_size = kernel.threads_per_cta(); + unsigned max_cta_per_sm = num_threads/cta_size; // e.g., 256 / 48 = 5 + assert( max_cta_per_sm > 0 ); - if (*thread_info != NULL) { - ptx_thread_info *thd = *thread_info; - assert(thd->is_done()); - if (g_debug_execution == -1) { - dim3 ctaid = thd->get_ctaid(); - dim3 t = thd->get_tid(); - printf( - "GPGPU-Sim PTX simulator: thread exiting ctaid=(%u,%u,%u) " - "tid=(%u,%u,%u) uid=%u\n", - ctaid.x, ctaid.y, ctaid.z, t.x, t.y, t.z, thd->get_uid()); - fflush(stdout); - } - thd->m_cta_info->register_deleted_thread(thd); - delete thd; - *thread_info = NULL; - } - - if (!active_threads.empty()) { - assert(active_threads.size() <= threads_left); - ptx_thread_info *thd = active_threads.front(); - active_threads.pop_front(); - *thread_info = thd; - thd->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, - isInFunctionalSimulationMode); - return 1; - } - - if (kernel.no_more_ctas_to_run()) { - return 0; // finished! - } - - if (threads_left < kernel.threads_per_cta()) { - return 0; - } - - if (g_debug_execution == -1) { - printf("GPGPU-Sim PTX simulator: STARTING THREAD ALLOCATION --> \n"); - fflush(stdout); - } - - // initializing new CTA - ptx_cta_info *cta_info = NULL; - memory_space *shared_mem = NULL; - memory_space *sstarr_mem = NULL; - - unsigned cta_size = kernel.threads_per_cta(); - unsigned max_cta_per_sm = num_threads / cta_size; // e.g., 256 / 48 = 5 - assert(max_cta_per_sm > 0); - - // unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid; - unsigned sm_idx = - hw_cta_id * gpu->gpgpu_ctx->func_sim->gpgpu_param_num_shaders + sid; - - if (shared_memory_lookup.find(sm_idx) == shared_memory_lookup.end()) { - if (g_debug_execution >= 1) { - printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", sm_idx, - sid, max_cta_per_sm); - } - char buf[512]; - snprintf(buf, 512, "shared_%u", sid); - shared_mem = new memory_space_impl<16 * 1024>(buf, 4); - shared_memory_lookup[sm_idx] = shared_mem; - snprintf(buf, 512, "sstarr_%u", sid); - sstarr_mem = new memory_space_impl<16 * 1024>(buf, 4); - sstarr_memory_lookup[sm_idx] = sstarr_mem; - cta_info = new ptx_cta_info(sm_idx, gpu->gpgpu_ctx); - ptx_cta_lookup[sm_idx] = cta_info; - } else { - if (g_debug_execution >= 1) { - printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", sm_idx, - sid, max_cta_per_sm); - } - shared_mem = shared_memory_lookup[sm_idx]; - sstarr_mem = sstarr_memory_lookup[sm_idx]; - cta_info = ptx_cta_lookup[sm_idx]; - cta_info->check_cta_thread_status_and_reset(); - } - - std::map &local_mem_lookup = - local_memory_lookup[sid]; - while (kernel.more_threads_in_cta()) { - dim3 ctaid3d = kernel.get_next_cta_id(); - unsigned new_tid = kernel.get_next_thread_id(); - dim3 tid3d = kernel.get_next_thread_id_3d(); - kernel.increment_thread_id(); - new_tid += tid; - ptx_thread_info *thd = new ptx_thread_info(kernel); - ptx_warp_info *warp_info = NULL; - if (ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end()) { - warp_info = new ptx_warp_info(); - ptx_warp_lookup[hw_warp_id] = warp_info; - } else { - warp_info = ptx_warp_lookup[hw_warp_id]; - } - thd->m_warp_info = warp_info; + //unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid; + unsigned sm_idx = hw_cta_id*gpu->gpgpu_ctx->func_sim->gpgpu_param_num_shaders + sid; - memory_space *local_mem = NULL; - std::map::iterator l = - local_mem_lookup.find(new_tid); - if (l != local_mem_lookup.end()) { - local_mem = l->second; - } else { + if ( shared_memory_lookup.find(sm_idx) == shared_memory_lookup.end() ) { + if ( g_debug_execution >= 1 ) { + printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", + sm_idx, sid, max_cta_per_sm ); + } char buf[512]; - snprintf(buf, 512, "local_%u_%u", sid, new_tid); - local_mem = new memory_space_impl<32>(buf, 32); - local_mem_lookup[new_tid] = local_mem; - } - thd->set_info(kernel.entry()); - thd->set_nctaid(kernel.get_grid_dim()); - thd->set_ntid(kernel.get_cta_dim()); - thd->set_ctaid(ctaid3d); - thd->set_tid(tid3d); - if (kernel.entry()->get_ptx_version().extensions()) - thd->cpy_tid_to_reg(tid3d); - thd->set_valid(); - thd->m_shared_mem = shared_mem; - thd->m_sstarr_mem = sstarr_mem; - function_info *finfo = thd->func_info(); - symbol_table *st = finfo->get_symtab(); - thd->func_info()->param_to_shared(thd->m_shared_mem, st); - thd->func_info()->param_to_shared(thd->m_sstarr_mem, st); - thd->m_cta_info = cta_info; - cta_info->add_thread(thd); - thd->m_local_mem = local_mem; - if (g_debug_execution == -1) { - printf( - "GPGPU-Sim PTX simulator: allocating thread ctaid=(%u,%u,%u) " - "tid=(%u,%u,%u) @ 0x%Lx\n", - ctaid3d.x, ctaid3d.y, ctaid3d.z, tid3d.x, tid3d.y, tid3d.z, - (unsigned long long)thd); + snprintf(buf,512,"shared_%u", sid); + shared_mem = new memory_space_impl<16*1024>(buf,4); + shared_memory_lookup[sm_idx] = shared_mem; + snprintf(buf,512,"sstarr_%u", sid); + sstarr_mem = new memory_space_impl<16*1024>(buf,4); + sstarr_memory_lookup[sm_idx] = sstarr_mem; + cta_info = new ptx_cta_info(sm_idx, gpu->gpgpu_ctx); + ptx_cta_lookup[sm_idx] = cta_info; + } else { + if ( g_debug_execution >= 1 ) { + printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", + sm_idx, sid, max_cta_per_sm ); + } + shared_mem = shared_memory_lookup[sm_idx]; + sstarr_mem = sstarr_memory_lookup[sm_idx]; + cta_info = ptx_cta_lookup[sm_idx]; + cta_info->check_cta_thread_status_and_reset(); + } + + std::map &local_mem_lookup = local_memory_lookup[sid]; + while( kernel.more_threads_in_cta() ) { + dim3 ctaid3d = kernel.get_next_cta_id(); + unsigned new_tid = kernel.get_next_thread_id(); + dim3 tid3d = kernel.get_next_thread_id_3d(); + kernel.increment_thread_id(); + new_tid += tid; + ptx_thread_info *thd = new ptx_thread_info(kernel); + ptx_warp_info *warp_info = NULL; + if ( ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end() ) { + warp_info = new ptx_warp_info(); + ptx_warp_lookup[hw_warp_id] = warp_info; + } else { + warp_info = ptx_warp_lookup[hw_warp_id]; + } + thd->m_warp_info = warp_info; + + memory_space *local_mem = NULL; + std::map::iterator l = local_mem_lookup.find(new_tid); + if ( l != local_mem_lookup.end() ) { + local_mem = l->second; + } else { + char buf[512]; + snprintf(buf,512,"local_%u_%u", sid, new_tid); + local_mem = new memory_space_impl<32>(buf,32); + local_mem_lookup[new_tid] = local_mem; + } + thd->set_info(kernel.entry()); + thd->set_nctaid(kernel.get_grid_dim()); + thd->set_ntid(kernel.get_cta_dim()); + thd->set_ctaid(ctaid3d); + thd->set_tid(tid3d); + if( kernel.entry()->get_ptx_version().extensions() ) + thd->cpy_tid_to_reg(tid3d); + thd->set_valid(); + thd->m_shared_mem = shared_mem; + thd->m_sstarr_mem = sstarr_mem; + function_info *finfo = thd->func_info(); + symbol_table *st = finfo->get_symtab(); + thd->func_info()->param_to_shared(thd->m_shared_mem,st); + thd->func_info()->param_to_shared(thd->m_sstarr_mem,st); + thd->m_cta_info = cta_info; + cta_info->add_thread(thd); + thd->m_local_mem = local_mem; + if ( g_debug_execution==-1 ) { + printf("GPGPU-Sim PTX simulator: allocating thread ctaid=(%u,%u,%u) tid=(%u,%u,%u) @ 0x%Lx\n", + ctaid3d.x,ctaid3d.y,ctaid3d.z,tid3d.x,tid3d.y,tid3d.z, (unsigned long long)thd ); + fflush(stdout); + } + active_threads.push_back(thd); + } + if ( g_debug_execution==-1 ) { + printf("GPGPU-Sim PTX simulator: <-- FINISHING THREAD ALLOCATION\n"); fflush(stdout); - } - active_threads.push_back(thd); - } - if (g_debug_execution == -1) { - printf("GPGPU-Sim PTX simulator: <-- FINISHING THREAD ALLOCATION\n"); - fflush(stdout); - } - - kernel.increment_cta_id(); - - assert(active_threads.size() <= threads_left); - *thread_info = active_threads.front(); - (*thread_info) - ->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, - isInFunctionalSimulationMode); - active_threads.pop_front(); - return 1; -} - -size_t get_kernel_code_size(class function_info *entry) { - return entry->get_function_size(); -} - -kernel_info_t *cuda_sim::gpgpu_opencl_ptx_sim_init_grid( - class function_info *entry, gpgpu_ptx_sim_arg_list_t args, - struct dim3 gridDim, struct dim3 blockDim, gpgpu_t *gpu) { - kernel_info_t *result = - new kernel_info_t(gridDim, blockDim, entry, gpu->getNameArrayMapping(), - gpu->getNameInfoMapping()); - unsigned argcount = args.size(); - unsigned argn = 1; - for (gpgpu_ptx_sim_arg_list_t::iterator a = args.begin(); a != args.end(); - a++) { - entry->add_param_data(argcount - argn, &(*a)); - argn++; - } - entry->finalize(result->get_param_memory()); - g_ptx_kernel_count++; - fflush(stdout); - - return result; + } + + kernel.increment_cta_id(); + + assert( active_threads.size() <= threads_left ); + *thread_info = active_threads.front(); + (*thread_info)->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid,isInFunctionalSimulationMode ); + active_threads.pop_front(); + return 1; +} + +size_t get_kernel_code_size( class function_info *entry ) +{ + return entry->get_function_size(); +} + + +kernel_info_t *cuda_sim::gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, + gpgpu_ptx_sim_arg_list_t args, + struct dim3 gridDim, + struct dim3 blockDim, + gpgpu_t *gpu ) +{ + kernel_info_t *result = new kernel_info_t(gridDim,blockDim,entry,gpu->getNameArrayMapping(),gpu->getNameInfoMapping()); + unsigned argcount=args.size(); + unsigned argn=1; + for( gpgpu_ptx_sim_arg_list_t::iterator a = args.begin(); a != args.end(); a++ ) { + entry->add_param_data(argcount-argn,&(*a)); + argn++; + } + entry->finalize(result->get_param_memory()); + g_ptx_kernel_count++; + fflush(stdout); + + return result; } #include "../../version" #include "detailed_version" -void print_splash() { - static int splash_printed = 0; - if (!splash_printed) { - fprintf(stdout, "\n\n *** %s [build %s] ***\n\n\n", - g_gpgpusim_version_string, g_gpgpusim_build_string); - splash_printed = 1; - } -} - -void cuda_sim::gpgpu_ptx_sim_register_const_variable(void *hostVar, - const char *deviceName, - size_t size) { - printf("GPGPU-Sim PTX registering constant %s (%zu bytes) to name mapping\n", - deviceName, size); - g_const_name_lookup[hostVar] = deviceName; -} - -void cuda_sim::gpgpu_ptx_sim_register_global_variable(void *hostVar, - const char *deviceName, - size_t size) { - printf("GPGPU-Sim PTX registering global %s hostVar to name mapping\n", - deviceName); - g_global_name_lookup[hostVar] = deviceName; -} - -void cuda_sim::gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, - size_t count, size_t offset, int to, - gpgpu_t *gpu) { - printf( - "GPGPU-Sim PTX: starting gpgpu_ptx_sim_memcpy_symbol with hostVar 0x%p\n", - hostVar); - bool found_sym = false; - memory_space_t mem_region = undefined_space; - std::string sym_name; - - std::map::iterator c = - gpu->gpgpu_ctx->func_sim->g_const_name_lookup.find(hostVar); - if (c != gpu->gpgpu_ctx->func_sim->g_const_name_lookup.end()) { - found_sym = true; - sym_name = c->second; - mem_region = const_space; - } - std::map::iterator g = - gpu->gpgpu_ctx->func_sim->g_global_name_lookup.find(hostVar); - if (g != gpu->gpgpu_ctx->func_sim->g_global_name_lookup.end()) { - if (found_sym) { - printf( - "Execution error: PTX symbol \"%s\" w/ hostVar=0x%Lx is declared " - "both const and global?\n", - sym_name.c_str(), (unsigned long long)hostVar); +void print_splash() +{ + static int splash_printed=0; + if ( !splash_printed ) { + fprintf(stdout, "\n\n *** %s [build %s] ***\n\n\n", g_gpgpusim_version_string, g_gpgpusim_build_string ); + splash_printed=1; + } +} + +void cuda_sim::gpgpu_ptx_sim_register_const_variable(void *hostVar, const char *deviceName, size_t size ) +{ + printf("GPGPU-Sim PTX registering constant %s (%zu bytes) to name mapping\n", deviceName, size ); + g_const_name_lookup[hostVar] = deviceName; +} + +void cuda_sim::gpgpu_ptx_sim_register_global_variable(void *hostVar, const char *deviceName, size_t size ) +{ + printf("GPGPU-Sim PTX registering global %s hostVar to name mapping\n", deviceName ); + g_global_name_lookup[hostVar] = deviceName; +} + +void cuda_sim::gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to, gpgpu_t *gpu ) +{ + printf("GPGPU-Sim PTX: starting gpgpu_ptx_sim_memcpy_symbol with hostVar 0x%p\n", hostVar); + bool found_sym = false; + memory_space_t mem_region = undefined_space; + std::string sym_name; + + std::map::iterator c=gpu->gpgpu_ctx->func_sim->g_const_name_lookup.find(hostVar); + if ( c!=gpu->gpgpu_ctx->func_sim->g_const_name_lookup.end() ) { + found_sym = true; + sym_name = c->second; + mem_region = const_space; + } + std::map::iterator g=gpu->gpgpu_ctx->func_sim->g_global_name_lookup.find(hostVar); + if ( g!=gpu->gpgpu_ctx->func_sim->g_global_name_lookup.end() ) { + if ( found_sym ) { + printf("Execution error: PTX symbol \"%s\" w/ hostVar=0x%Lx is declared both const and global?\n", + sym_name.c_str(), (unsigned long long)hostVar ); + abort(); + } + found_sym = true; + sym_name = g->second; + mem_region = global_space; + } + if( g_globals.find(hostVar) != g_globals.end() ) { + found_sym = true; + sym_name = hostVar; + mem_region = global_space; + } + if( g_constants.find(hostVar) != g_constants.end() ) { + found_sym = true; + sym_name = hostVar; + mem_region = const_space; + } + + if ( !found_sym ) { + printf("Execution error: No information for PTX symbol w/ hostVar=0x%Lx\n", (unsigned long long)hostVar ); abort(); - } - found_sym = true; - sym_name = g->second; - mem_region = global_space; - } - if (g_globals.find(hostVar) != g_globals.end()) { - found_sym = true; - sym_name = hostVar; - mem_region = global_space; - } - if (g_constants.find(hostVar) != g_constants.end()) { - found_sym = true; - sym_name = hostVar; - mem_region = const_space; - } - - if (!found_sym) { - printf("Execution error: No information for PTX symbol w/ hostVar=0x%Lx\n", - (unsigned long long)hostVar); - abort(); - } else - printf( - "GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: Found PTX symbol w/ " - "hostVar=0x%Lx\n", - (unsigned long long)hostVar); - const char *mem_name = NULL; - memory_space *mem = NULL; - - std::map::iterator st = - gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.find(sym_name.c_str()); - assert(st != gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.end()); - symbol_table *symtab = st->second; - - symbol *sym = symtab->lookup(sym_name.c_str()); - assert(sym); - unsigned dst = sym->get_address() + offset; - switch (mem_region.get_type()) { - case const_space: + } else printf("GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: Found PTX symbol w/ hostVar=0x%Lx\n", (unsigned long long)hostVar ); + const char *mem_name = NULL; + memory_space *mem = NULL; + + std::map::iterator st = gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.find(sym_name.c_str()); + assert( st != gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.end() ); + symbol_table *symtab = st->second; + + symbol *sym = symtab->lookup(sym_name.c_str()); + assert(sym); + unsigned dst = sym->get_address() + offset; + switch (mem_region.get_type()) { + case const_space: mem = gpu->get_global_memory(); mem_name = "const"; break; - case global_space: + case global_space: mem = gpu->get_global_memory(); mem_name = "global"; break; - default: + default: abort(); - } - printf( - "GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: copying %s memory %zu bytes " - "%s symbol %s+%zu @0x%x ...\n", - mem_name, count, (to ? " to " : "from"), sym_name.c_str(), offset, dst); - for (unsigned n = 0; n < count; n++) { - if (to) - mem->write(dst + n, 1, ((char *)src) + n, NULL, NULL); - else - mem->read(dst + n, 1, ((char *)src) + n); - } - fflush(stdout); + } + printf("GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: copying %s memory %zu bytes %s symbol %s+%zu @0x%x ...\n", + mem_name, count, (to?" to ":"from"), sym_name.c_str(), offset, dst ); + for ( unsigned n=0; n < count; n++ ) { + if( to ) mem->write(dst+n,1,((char*)src)+n,NULL,NULL); + else mem->read(dst+n,1,((char*)src)+n); + } + fflush(stdout); } extern int ptx_debug; -void cuda_sim::read_sim_environment_variables() { - ptx_debug = 0; - g_debug_execution = 0; - g_interactive_debugger_enabled = false; - - char *mode = getenv("PTX_SIM_MODE_FUNC"); - if (mode) sscanf(mode, "%u", &g_ptx_sim_mode); - printf( - "GPGPU-Sim PTX: simulation mode %d (can change with PTX_SIM_MODE_FUNC " - "environment variable:\n", - g_ptx_sim_mode); - printf( - " 1=functional simulation only, 0=detailed performance " - "simulator)\n"); - char *dbg_inter = getenv("GPGPUSIM_DEBUG"); - if (dbg_inter && strlen(dbg_inter)) { - printf("GPGPU-Sim PTX: enabling interactive debugger\n"); - fflush(stdout); - g_interactive_debugger_enabled = true; - } - char *dbg_level = getenv("PTX_SIM_DEBUG"); - if (dbg_level && strlen(dbg_level)) { - printf("GPGPU-Sim PTX: setting debug level to %s\n", dbg_level); - fflush(stdout); - sscanf(dbg_level, "%d", &g_debug_execution); - } - char *dbg_thread = getenv("PTX_SIM_DEBUG_THREAD_UID"); - if (dbg_thread && strlen(dbg_thread)) { - printf("GPGPU-Sim PTX: printing debug information for thread uid %s\n", - dbg_thread); - fflush(stdout); - sscanf(dbg_thread, "%d", &g_debug_thread_uid); - } - char *dbg_pc = getenv("PTX_SIM_DEBUG_PC"); - if (dbg_pc && strlen(dbg_pc)) { - printf( - "GPGPU-Sim PTX: printing debug information for instruction with PC = " - "%s\n", - dbg_pc); - fflush(stdout); - sscanf(dbg_pc, "%d", &g_debug_pc); - } +void cuda_sim::read_sim_environment_variables() +{ + ptx_debug = 0; + g_debug_execution = 0; + g_interactive_debugger_enabled = false; + + char *mode = getenv("PTX_SIM_MODE_FUNC"); + if ( mode ) + sscanf(mode,"%u", &g_ptx_sim_mode); + printf("GPGPU-Sim PTX: simulation mode %d (can change with PTX_SIM_MODE_FUNC environment variable:\n", g_ptx_sim_mode); + printf(" 1=functional simulation only, 0=detailed performance simulator)\n"); + char *dbg_inter = getenv("GPGPUSIM_DEBUG"); + if ( dbg_inter && strlen(dbg_inter) ) { + printf("GPGPU-Sim PTX: enabling interactive debugger\n"); + fflush(stdout); + g_interactive_debugger_enabled = true; + } + char *dbg_level = getenv("PTX_SIM_DEBUG"); + if ( dbg_level && strlen(dbg_level) ) { + printf("GPGPU-Sim PTX: setting debug level to %s\n", dbg_level ); + fflush(stdout); + sscanf(dbg_level,"%d", &g_debug_execution); + } + char *dbg_thread = getenv("PTX_SIM_DEBUG_THREAD_UID"); + if ( dbg_thread && strlen(dbg_thread) ) { + printf("GPGPU-Sim PTX: printing debug information for thread uid %s\n", dbg_thread ); + fflush(stdout); + sscanf(dbg_thread,"%d", &g_debug_thread_uid); + } + char *dbg_pc = getenv("PTX_SIM_DEBUG_PC"); + if ( dbg_pc && strlen(dbg_pc) ) { + printf("GPGPU-Sim PTX: printing debug information for instruction with PC = %s\n", dbg_pc ); + fflush(stdout); + sscanf(dbg_pc,"%d", &g_debug_pc); + } #if CUDART_VERSION > 1010 - g_override_embedded_ptx = false; - char *usefile = getenv("PTX_SIM_USE_PTX_FILE"); - if (usefile && strlen(usefile)) { - printf( - "GPGPU-Sim PTX: overriding embedded ptx with ptx file " - "(PTX_SIM_USE_PTX_FILE is set)\n"); - fflush(stdout); - g_override_embedded_ptx = true; - } - char *blocking = getenv("CUDA_LAUNCH_BLOCKING"); - if (blocking && !strcmp(blocking, "1")) { - g_cuda_launch_blocking = true; - } + g_override_embedded_ptx = false; + char *usefile = getenv("PTX_SIM_USE_PTX_FILE"); + if (usefile && strlen(usefile)) { + printf("GPGPU-Sim PTX: overriding embedded ptx with ptx file (PTX_SIM_USE_PTX_FILE is set)\n"); + fflush(stdout); + g_override_embedded_ptx = true; + } + char *blocking = getenv("CUDA_LAUNCH_BLOCKING"); + if( blocking && !strcmp(blocking,"1") ) { + g_cuda_launch_blocking = true; + } #else - g_cuda_launch_blocking = true; - g_override_embedded_ptx = true; + g_cuda_launch_blocking = true; + g_override_embedded_ptx = true; #endif - if (g_debug_execution >= 40) { - ptx_debug = 1; - } -} - -#define MAX(a, b) (((a) > (b)) ? (a) : (b)) - -unsigned max_cta(const struct gpgpu_ptx_sim_info *kernel_info, - unsigned threads_per_cta, unsigned int warp_size, - unsigned int n_thread_per_shader, - unsigned int gpgpu_shmem_size, - unsigned int gpgpu_shader_registers, - unsigned int max_cta_per_core) { - unsigned int padded_cta_size = threads_per_cta; - if (padded_cta_size % warp_size) - padded_cta_size = ((padded_cta_size / warp_size) + 1) * (warp_size); - unsigned int result_thread = n_thread_per_shader / padded_cta_size; - - unsigned int result_shmem = (unsigned)-1; - if (kernel_info->smem > 0) - result_shmem = gpgpu_shmem_size / kernel_info->smem; - unsigned int result_regs = (unsigned)-1; - if (kernel_info->regs > 0) - result_regs = gpgpu_shader_registers / - (padded_cta_size * ((kernel_info->regs + 3) & ~3)); - printf("padded cta size is %d and %d and %d", padded_cta_size, - kernel_info->regs, ((kernel_info->regs + 3) & ~3)); - // Limit by CTA - unsigned int result_cta = max_cta_per_core; - - unsigned result = result_thread; - result = gs_min2(result, result_shmem); - result = gs_min2(result, result_regs); - result = gs_min2(result, result_cta); - - printf("GPGPU-Sim uArch: CTA/core = %u, limited by:", result); - if (result == result_thread) printf(" threads"); - if (result == result_shmem) printf(" shmem"); - if (result == result_regs) printf(" regs"); - if (result == result_cta) printf(" cta_limit"); - printf("\n"); - - return result; + if ( g_debug_execution >= 40 ) { + ptx_debug = 1; + } +} + +#define MAX(a,b) (((a)>(b))?(a):(b)) + +unsigned max_cta (const struct gpgpu_ptx_sim_info *kernel_info, unsigned threads_per_cta, unsigned int warp_size, unsigned int n_thread_per_shader, unsigned int gpgpu_shmem_size, unsigned int gpgpu_shader_registers, unsigned int max_cta_per_core) +{ + + unsigned int padded_cta_size = threads_per_cta; + if (padded_cta_size%warp_size) + padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size); + unsigned int result_thread = n_thread_per_shader / padded_cta_size; + + unsigned int result_shmem = (unsigned)-1; + if (kernel_info->smem > 0) + result_shmem = gpgpu_shmem_size / kernel_info->smem; + unsigned int result_regs = (unsigned)-1; + if (kernel_info->regs > 0) + result_regs = gpgpu_shader_registers / (padded_cta_size * ((kernel_info->regs+3)&~3)); + printf("padded cta size is %d and %d and %d",padded_cta_size, kernel_info->regs, ((kernel_info->regs+3)&~3) ); + //Limit by CTA + unsigned int result_cta = max_cta_per_core; + + unsigned result = result_thread; + result = gs_min2(result, result_shmem); + result = gs_min2(result, result_regs); + result = gs_min2(result, result_cta); + + printf ("GPGPU-Sim uArch: CTA/core = %u, limited by:", result); + if (result == result_thread) printf (" threads"); + if (result == result_shmem) printf (" shmem"); + if (result == result_regs) printf (" regs"); + if (result == result_cta) printf (" cta_limit"); + printf ("\n"); + + return result; } /*! -This function simulates the CUDA code functionally, it takes a kernel_info_t -parameter +This function simulates the CUDA code functionally, it takes a kernel_info_t parameter which holds the data for the CUDA kernel to be executed !*/ -void cuda_sim::gpgpu_cuda_ptx_sim_main_func(kernel_info_t &kernel, - bool openCL) { - printf( - "GPGPU-Sim: Performing Functional Simulation, executing kernel %s...\n", - kernel.name().c_str()); - - // using a shader core object for book keeping, it is not needed but as most - // function built for performance simulation need it we use it here - // extern gpgpu_sim *g_the_gpu; - // before we execute, we should do PDOM analysis for functional simulation - // scenario. - function_info *kernel_func_info = kernel.entry(); - const struct gpgpu_ptx_sim_info *kernel_info = - ptx_sim_kernel_info(kernel_func_info); - checkpoint *g_checkpoint; - g_checkpoint = new checkpoint(); - - if (kernel_func_info->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", - kernel.name().c_str()); - } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", - kernel.name().c_str()); - kernel_func_info->do_pdom(); - kernel_func_info->set_pdom(); - } - - unsigned max_cta_tot = max_cta( - kernel_info, kernel.threads_per_cta(), - gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size, - gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() - ->n_thread_per_shader, - gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() - ->gpgpu_shmem_size, - gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() - ->gpgpu_shader_registers, - gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() - ->max_cta_per_core); - printf("Max CTA : %d\n", max_cta_tot); - - int cp_op = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_option; - int cp_kernel = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_kernel; - cp_count = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_insn_Y; - cp_cta_resume = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_CTA_t; - int cta_launched = 0; - - // we excute the kernel one CTA (Block) at the time, as synchronization - // functions work block wise - while (!kernel.no_more_ctas_to_run()) { - unsigned temp = kernel.get_next_cta_id_single(); - - if (cp_op == 0 || (cp_op == 1 && cta_launched < cp_cta_resume && - kernel.get_uid() == cp_kernel) || - kernel.get_uid() < cp_kernel) // just fro testing - { - functionalCoreSim cta( - &kernel, gpgpu_ctx->the_gpgpusim->g_the_gpu, - gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size); - cta.execute(cp_count, temp); - -#if (CUDART_VERSION >= 5000) - gpgpu_ctx->device_runtime->launch_all_device_kernels(); -#endif +void cuda_sim::gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) +{ + printf("GPGPU-Sim: Performing Functional Simulation, executing kernel %s...\n",kernel.name().c_str()); + + //using a shader core object for book keeping, it is not needed but as most function built for performance simulation need it we use it here + //extern gpgpu_sim *g_the_gpu; + //before we execute, we should do PDOM analysis for functional simulation scenario. + function_info *kernel_func_info = kernel.entry(); + const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel_func_info); + checkpoint *g_checkpoint; + g_checkpoint = new checkpoint(); + + if (kernel_func_info->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", kernel.name().c_str() ); } else { - kernel.increment_cta_id(); + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", kernel.name().c_str() ); + kernel_func_info->do_pdom(); + kernel_func_info->set_pdom(); } + + unsigned max_cta_tot = max_cta(kernel_info,kernel.threads_per_cta(), gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->n_thread_per_shader, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->gpgpu_shmem_size, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->gpgpu_shader_registers, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->max_cta_per_core); + printf("Max CTA : %d\n",max_cta_tot); + + int cp_op= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_option; + int cp_kernel= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_kernel; + cp_count= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_insn_Y; + cp_cta_resume= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_CTA_t; + int cta_launched =0; + + //we excute the kernel one CTA (Block) at the time, as synchronization functions work block wise + while(!kernel.no_more_ctas_to_run()){ + unsigned temp=kernel.get_next_cta_id_single(); + + + if(cp_op==0 || (cp_op==1 && cta_launchedthe_gpgpusim->g_the_gpu, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size + ); + cta.execute(cp_count,temp); + + #if (CUDART_VERSION >= 5000) + gpgpu_ctx->device_runtime->launch_all_device_kernels(); + #endif + } + else + { + kernel.increment_cta_id(); + } cta_launched++; - } - - if (cp_op == 1) { - char f1name[2048]; - snprintf(f1name, 2048, "checkpoint_files/global_mem_%d.txt", - kernel.get_uid()); - g_checkpoint->store_global_mem( - gpgpu_ctx->the_gpgpusim->g_the_gpu->get_global_memory(), f1name, - (char *)"%08x"); - } - - // registering this kernel as done - - // openCL kernel simulation calls don't register the kernel so we don't - // register its exit - if (!openCL) { - // extern stream_manager *g_stream_manager; - gpgpu_ctx->the_gpgpusim->g_stream_manager->register_finished_kernel( - kernel.get_uid()); - } - - //******PRINTING******* - printf("GPGPU-Sim: Done functional simulation (%u instructions simulated).\n", - g_ptx_sim_num_insn); - if (gpgpu_ptx_instruction_classification) { - StatDisp(g_inst_classification_stat[g_ptx_kernel_count]); - StatDisp(g_inst_op_classification_stat[g_ptx_kernel_count]); - } - - // time_t variables used to calculate the total simulation time - // the start time of simulation is hold by the global variable - // g_simulation_starttime - // g_simulation_starttime is initilized by gpgpu_ptx_sim_init_perf() in - // gpgpusim_entrypoint.cc upon starting gpgpu-sim - time_t end_time, elapsed_time, days, hrs, minutes, sec; - end_time = time((time_t *)NULL); - elapsed_time = - MAX(end_time - gpgpu_ctx->the_gpgpusim->g_simulation_starttime, 1); - - // calculating and printing simulation time in terms of days, hours, minutes - // and seconds - days = elapsed_time / (3600 * 24); - hrs = elapsed_time / 3600 - 24 * days; - minutes = elapsed_time / 60 - 60 * (hrs + 24 * days); - sec = elapsed_time - 60 * (minutes + 60 * (hrs + 24 * days)); - - fflush(stderr); - printf( - "\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", - (unsigned)days, (unsigned)hrs, (unsigned)minutes, (unsigned)sec, - (unsigned)elapsed_time); - printf("gpgpu_simulation_rate = %u (inst/sec)\n", - (unsigned)(g_ptx_sim_num_insn / elapsed_time)); - fflush(stdout); -} - -void functionalCoreSim::initializeCTA(unsigned ctaid_cp) { - int ctaLiveThreads = 0; - symbol_table *symtab = m_kernel->entry()->get_symtab(); - - for (int i = 0; i < m_warp_count; i++) { - m_warpAtBarrier[i] = false; - m_liveThreadCount[i] = 0; - } - for (int i = 0; i < m_warp_count * m_warp_size; i++) m_thread[i] = NULL; - - // get threads for a cta - for (unsigned i = 0; i < m_kernel->threads_per_cta(); i++) { - ptx_sim_init_thread(*m_kernel, &m_thread[i], 0, i, - m_kernel->threads_per_cta() - i, - m_kernel->threads_per_cta(), this, 0, i / m_warp_size, - (gpgpu_t *)m_gpu, true); - assert(m_thread[i] != NULL && !m_thread[i]->is_done()); - char fname[2048]; - snprintf(fname, 2048, "checkpoint_files/thread_%d_0_reg.txt", i); - if (m_gpu->gpgpu_ctx->func_sim->cp_cta_resume == 1) - m_thread[i]->resume_reg_thread(fname, symtab); - ctaLiveThreads++; - } - - for (int k = 0; k < m_warp_count; k++) createWarp(k); -} - -void functionalCoreSim::createWarp(unsigned warpId) { - simt_mask_t initialMask; - unsigned liveThreadsCount = 0; - initialMask.set(); - for (int i = warpId * m_warp_size; i < warpId * m_warp_size + m_warp_size; - i++) { - if (m_thread[i] == NULL) - initialMask.reset(i - warpId * m_warp_size); - else - liveThreadsCount++; - } - - assert(m_thread[warpId * m_warp_size] != NULL); - m_simt_stack[warpId]->launch(m_thread[warpId * m_warp_size]->get_pc(), - initialMask); - char fname[2048]; - snprintf(fname, 2048, "checkpoint_files/warp_%d_0_simt.txt", warpId); - - if (m_gpu->gpgpu_ctx->func_sim->cp_cta_resume == 1) { - unsigned pc, rpc; - m_simt_stack[warpId]->resume(fname); - m_simt_stack[warpId]->get_pdom_stack_top_info(&pc, &rpc); - for (int i = warpId * m_warp_size; i < warpId * m_warp_size + m_warp_size; - i++) { - m_thread[i]->set_npc(pc); - m_thread[i]->update_pc(); - } - } - m_liveThreadCount[warpId] = liveThreadsCount; -} - -void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp) { - m_gpu->gpgpu_ctx->func_sim->cp_count = m_gpu->checkpoint_insn_Y; - m_gpu->gpgpu_ctx->func_sim->cp_cta_resume = m_gpu->checkpoint_CTA_t; - initializeCTA(ctaid_cp); - - int count = 0; - while (true) { - bool someOneLive = false; - bool allAtBarrier = true; - for (unsigned i = 0; i < m_warp_count; i++) { - executeWarp(i, allAtBarrier, someOneLive); - count++; } - if (inst_count > 0 && count > inst_count && - (m_kernel->get_uid() == m_gpu->checkpoint_kernel) && - (ctaid_cp >= m_gpu->checkpoint_CTA) && - (ctaid_cp < m_gpu->checkpoint_CTA_t) && m_gpu->checkpoint_option == 1) { - someOneLive = false; - break; - } - if (!someOneLive) break; - if (allAtBarrier) { - for (unsigned i = 0; i < m_warp_count; i++) m_warpAtBarrier[i] = false; - } - } - - checkpoint *g_checkpoint; - g_checkpoint = new checkpoint(); - - ptx_reg_t regval; - regval.u64 = 123; - - unsigned ctaid = m_kernel->get_next_cta_id_single(); - if (m_gpu->checkpoint_option == 1 && - (m_kernel->get_uid() == m_gpu->checkpoint_kernel) && - (ctaid_cp >= m_gpu->checkpoint_CTA) && - (ctaid_cp < m_gpu->checkpoint_CTA_t)) { - char fname[2048]; - snprintf(fname, 2048, "checkpoint_files/shared_mem_%d.txt", ctaid - 1); - g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname, - (char *)"%08x"); - for (int i = 0; i < 32 * m_warp_count; i++) { - char fname[2048]; - snprintf(fname, 2048, "checkpoint_files/thread_%d_%d_reg.txt", i, - ctaid - 1); - m_thread[i]->print_reg_thread(fname); + + + if(cp_op==1) + { char f1name[2048]; - snprintf(f1name, 2048, "checkpoint_files/local_mem_thread_%d_%d_reg.txt", - i, ctaid - 1); - g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name, - (char *)"%08x"); - m_thread[i]->set_done(); - m_thread[i]->exitCore(); - m_thread[i]->registerExit(); + snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", kernel.get_uid() ); + g_checkpoint->store_global_mem(gpgpu_ctx->the_gpgpusim->g_the_gpu->get_global_memory(), f1name , (char *)"%08x"); + } + + + + + //registering this kernel as done + + //openCL kernel simulation calls don't register the kernel so we don't register its exit + if(!openCL) { + //extern stream_manager *g_stream_manager; + gpgpu_ctx->the_gpgpusim->g_stream_manager->register_finished_kernel(kernel.get_uid()); + } + + //******PRINTING******* + printf( "GPGPU-Sim: Done functional simulation (%u instructions simulated).\n", g_ptx_sim_num_insn ); + if ( gpgpu_ptx_instruction_classification ) { + StatDisp( g_inst_classification_stat[g_ptx_kernel_count]); + StatDisp ( g_inst_op_classification_stat[g_ptx_kernel_count]); + } + + //time_t variables used to calculate the total simulation time + //the start time of simulation is hold by the global variable g_simulation_starttime + //g_simulation_starttime is initilized by gpgpu_ptx_sim_init_perf() in gpgpusim_entrypoint.cc upon starting gpgpu-sim + time_t end_time, elapsed_time, days, hrs, minutes, sec; + end_time = time((time_t *)NULL); + elapsed_time = MAX(end_time - gpgpu_ctx->the_gpgpusim->g_simulation_starttime, 1); + + + //calculating and printing simulation time in terms of days, hours, minutes and seconds + days = elapsed_time/(3600*24); + hrs = elapsed_time/3600 - 24*days; + minutes = elapsed_time/60 - 60*(hrs + 24*days); + sec = elapsed_time - 60*(minutes + 60*(hrs + 24*days)); + + fflush(stderr); + printf("\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", + (unsigned)days, (unsigned)hrs, (unsigned)minutes, (unsigned)sec, (unsigned)elapsed_time ); + printf("gpgpu_simulation_rate = %u (inst/sec)\n", (unsigned)(g_ptx_sim_num_insn / elapsed_time) ); + fflush(stdout); +} + +void functionalCoreSim::initializeCTA(unsigned ctaid_cp) +{ + int ctaLiveThreads=0; + symbol_table * symtab= m_kernel->entry()->get_symtab(); + + for(int i=0; i< m_warp_count; i++){ + m_warpAtBarrier[i]=false; + m_liveThreadCount[i]=0; } + for(int i=0; i< m_warp_count*m_warp_size;i++) + m_thread[i]=NULL; + + //get threads for a cta + for(unsigned i=0; ithreads_per_cta();i++) { + ptx_sim_init_thread(*m_kernel,&m_thread[i],0,i,m_kernel->threads_per_cta()-i,m_kernel->threads_per_cta(),this,0,i/m_warp_size,(gpgpu_t*)m_gpu, true); + assert(m_thread[i]!=NULL && !m_thread[i]->is_done()); + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/thread_%d_0_reg.txt",i ); + if(m_gpu->gpgpu_ctx->func_sim->cp_cta_resume==1) + m_thread[i]->resume_reg_thread(fname,symtab); + ctaLiveThreads++; + } + + for(int k=0;klaunch(m_thread[warpId*m_warp_size]->get_pc(),initialMask); + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/warp_%d_0_simt.txt",warpId ); + + if(m_gpu->gpgpu_ctx->func_sim->cp_cta_resume==1) + { + unsigned pc,rpc; + m_simt_stack[warpId]->resume(fname); + m_simt_stack[warpId]->get_pdom_stack_top_info(&pc,&rpc); + for(int i=warpId*m_warp_size; iset_npc(pc); + m_thread[i]->update_pc(); + } - for (int i = 0; i < m_warp_count; i++) { - char fname[2048]; - snprintf(fname, 2048, "checkpoint_files/warp_%d_%d_simt.txt", i, - ctaid - 1); - FILE *fp = fopen(fname, "w"); - assert(fp != NULL); - m_simt_stack[i]->print_checkpoint(fp); - fclose(fp); + } + m_liveThreadCount[warpId]= liveThreadsCount; +} + +void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp) + { + m_gpu->gpgpu_ctx->func_sim->cp_count= m_gpu->checkpoint_insn_Y; + m_gpu->gpgpu_ctx->func_sim->cp_cta_resume= m_gpu->checkpoint_CTA_t; + initializeCTA(ctaid_cp); + + int count=0; + while(true){ + bool someOneLive= false; + bool allAtBarrier = true; + for(unsigned i=0;i0 && count>inst_count && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cpcheckpoint_CTA_t) && m_gpu->checkpoint_option==1) + { + someOneLive=false; + break; + } + if(!someOneLive) break; + if(allAtBarrier){ + for(unsigned i=0;iget_next_cta_id_single(); + if(m_gpu->checkpoint_option==1 && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cpcheckpoint_CTA_t)) + { + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/shared_mem_%d.txt",ctaid-1 ); + g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname , (char *)"%08x"); + for(int i=0; i<32*m_warp_count;i++) + { + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i,ctaid-1 ); + m_thread[i]->print_reg_thread(fname); + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i,ctaid-1 ); + g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name , (char *)"%08x"); + m_thread[i]->set_done(); + m_thread[i]->exitCore(); + m_thread[i]->registerExit(); + } + + for(int i=0;iprint_checkpoint(fp); + fclose(fp); + } + } + } -void functionalCoreSim::executeWarp(unsigned i, bool &allAtBarrier, - bool &someOneLive) { - if (!m_warpAtBarrier[i] && m_liveThreadCount[i] != 0) { - warp_inst_t inst = getExecuteWarp(i); - execute_warp_inst_t(inst, i); - if (inst.isatomic()) inst.do_atomic(true); - if (inst.op == BARRIER_OP || inst.op == MEMORY_BARRIER_OP) - m_warpAtBarrier[i] = true; - updateSIMTStack(i, &inst); - } - if (m_liveThreadCount[i] > 0) someOneLive = true; - if (!m_warpAtBarrier[i] && m_liveThreadCount[i] > 0) allAtBarrier = false; +void functionalCoreSim::executeWarp(unsigned i, bool &allAtBarrier, bool & someOneLive) +{ + if(!m_warpAtBarrier[i] && m_liveThreadCount[i]!=0){ + warp_inst_t inst =getExecuteWarp(i); + execute_warp_inst_t(inst,i); + if(inst.isatomic()) inst.do_atomic(true); + if(inst.op==BARRIER_OP || inst.op==MEMORY_BARRIER_OP ) m_warpAtBarrier[i]=true; + updateSIMTStack( i, &inst ); + } + if(m_liveThreadCount[i]>0) someOneLive=true; + if(!m_warpAtBarrier[i]&& m_liveThreadCount[i]>0) allAtBarrier = false; } -unsigned gpgpu_context::translate_pc_to_ptxlineno(unsigned pc) { - // this function assumes that the kernel fits inside a single PTX file - // function_info *pFunc = g_func_info; // assume that the current kernel is - // the one in query - const ptx_instruction *pInsn = pc_to_instruction(pc); - unsigned ptx_line_number = pInsn->source_line(); +unsigned gpgpu_context::translate_pc_to_ptxlineno(unsigned pc) +{ + // this function assumes that the kernel fits inside a single PTX file + // function_info *pFunc = g_func_info; // assume that the current kernel is the one in query + const ptx_instruction *pInsn = pc_to_instruction(pc); + unsigned ptx_line_number = pInsn->source_line(); - return ptx_line_number; + return ptx_line_number; } // ptxinfo parser -extern std::map get_duplicate(); +extern std::map get_duplicate(); static char *g_ptxinfo_kname = NULL; static struct gpgpu_ptx_sim_info g_ptxinfo; -static std::map g_duplicate; +static std::map g_duplicate; static const char *g_last_dup_type; -const char *get_ptxinfo_kname() { return g_ptxinfo_kname; } +const char *get_ptxinfo_kname() +{ + return g_ptxinfo_kname; +} -void print_ptxinfo() { - if (!get_ptxinfo_kname()) { - printf("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", g_ptxinfo.gmem, - g_ptxinfo.cmem); - } - if (get_ptxinfo_kname()) { - printf( - "GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", - get_ptxinfo_kname(), g_ptxinfo.regs, g_ptxinfo.lmem, g_ptxinfo.smem, - g_ptxinfo.cmem); - } +void print_ptxinfo() +{ + if(! get_ptxinfo_kname()){ + printf ("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", + g_ptxinfo.gmem, + g_ptxinfo.cmem); + } + if(get_ptxinfo_kname()){ + printf ("GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", + get_ptxinfo_kname(), + g_ptxinfo.regs, + g_ptxinfo.lmem, + g_ptxinfo.smem, + g_ptxinfo.cmem ); + } } -struct gpgpu_ptx_sim_info get_ptxinfo() { - return g_ptxinfo; + +struct gpgpu_ptx_sim_info get_ptxinfo() +{ + return g_ptxinfo; } -std::map get_duplicate() { return g_duplicate; } +std::map get_duplicate() +{ + return g_duplicate; +} -void ptxinfo_linenum(unsigned linenum) { - g_duplicate[linenum] = g_last_dup_type; +void ptxinfo_linenum( unsigned linenum ) +{ + g_duplicate[linenum] = g_last_dup_type; } -void ptxinfo_dup_type(const char *dup_type) { g_last_dup_type = dup_type; } +void ptxinfo_dup_type( const char *dup_type ) +{ + g_last_dup_type = dup_type; +} -void ptxinfo_function(const char *fname) { - clear_ptxinfo(); - g_ptxinfo_kname = strdup(fname); +void ptxinfo_function(const char *fname ) +{ + clear_ptxinfo(); + g_ptxinfo_kname = strdup(fname); } -void ptxinfo_regs(unsigned nregs) { g_ptxinfo.regs = nregs; } +void ptxinfo_regs( unsigned nregs ) +{ + g_ptxinfo.regs=nregs; +} -void ptxinfo_lmem(unsigned declared, unsigned system) { - g_ptxinfo.lmem = declared + system; +void ptxinfo_lmem( unsigned declared, unsigned system ) +{ + g_ptxinfo.lmem=declared+system; } -void ptxinfo_gmem(unsigned declared, unsigned system) { - g_ptxinfo.gmem = declared + system; +void ptxinfo_gmem( unsigned declared, unsigned system ) +{ + g_ptxinfo.gmem=declared+system; } -void ptxinfo_smem(unsigned declared, unsigned system) { - g_ptxinfo.smem = declared + system; +void ptxinfo_smem( unsigned declared, unsigned system ) +{ + g_ptxinfo.smem=declared+system; } -void ptxinfo_cmem(unsigned nbytes, unsigned bank) { g_ptxinfo.cmem += nbytes; } +void ptxinfo_cmem( unsigned nbytes, unsigned bank ) +{ + g_ptxinfo.cmem+=nbytes; +} -void clear_ptxinfo() { - free(g_ptxinfo_kname); - g_ptxinfo_kname = NULL; - g_ptxinfo.regs = 0; - g_ptxinfo.lmem = 0; - g_ptxinfo.smem = 0; - g_ptxinfo.cmem = 0; - g_ptxinfo.gmem = 0; - g_ptxinfo.ptx_version = 0; - g_ptxinfo.sm_target = 0; +void clear_ptxinfo() +{ + free(g_ptxinfo_kname); + g_ptxinfo_kname=NULL; + g_ptxinfo.regs=0; + g_ptxinfo.lmem=0; + g_ptxinfo.smem=0; + g_ptxinfo.cmem=0; + g_ptxinfo.gmem=0; + g_ptxinfo.ptx_version=0; + g_ptxinfo.sm_target=0; } -void ptxinfo_opencl_addinfo(std::map &kernels) { - if (!g_ptxinfo_kname) { - printf("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", g_ptxinfo.gmem, - g_ptxinfo.cmem); - clear_ptxinfo(); - return; - } - if (!strcmp("__cuda_dummy_entry__", g_ptxinfo_kname)) { - // this string produced by ptxas for empty ptx files (e.g., bandwidth test) - clear_ptxinfo(); - return; - } - std::map::iterator k = - kernels.find(g_ptxinfo_kname); - if (k == kernels.end()) { - printf("GPGPU-Sim PTX: ERROR ** implementation for '%s' not found.\n", - g_ptxinfo_kname); - abort(); - } else { - printf( - "GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", - g_ptxinfo_kname, g_ptxinfo.regs, g_ptxinfo.lmem, g_ptxinfo.smem, - g_ptxinfo.cmem); - function_info *finfo = k->second; - assert(finfo != NULL); - finfo->set_kernel_info(g_ptxinfo); - } - clear_ptxinfo(); -} - -struct rec_pts cuda_sim::find_reconvergence_points(function_info *finfo) { - rec_pts tmp; - std::map::iterator r = g_rpts.find(finfo); - - if (r == g_rpts.end()) { - int num_recon = finfo->get_num_reconvergence_pairs(); - - gpgpu_recon_t *kernel_recon_points = - (struct gpgpu_recon_t *)calloc(num_recon, sizeof(struct gpgpu_recon_t)); - finfo->get_reconvergence_pairs(kernel_recon_points); - printf("GPGPU-Sim PTX: reconvergence points for %s...\n", - finfo->get_name().c_str()); - for (int i = 0; i < num_recon; i++) { - printf("GPGPU-Sim PTX: %2u (potential) branch divergence @ ", i + 1); - kernel_recon_points[i].source_inst->print_insn(); - printf("\n"); - printf("GPGPU-Sim PTX: immediate post dominator @ "); - if (kernel_recon_points[i].target_inst) - kernel_recon_points[i].target_inst->print_insn(); - printf("\n"); - } - printf("GPGPU-Sim PTX: ... end of reconvergence points for %s\n", - finfo->get_name().c_str()); - - tmp.s_kernel_recon_points = kernel_recon_points; - tmp.s_num_recon = num_recon; - g_rpts[finfo] = tmp; - } else { - tmp = r->second; - } - return tmp; -} - -address_type get_return_pc(void *thd) { - // function call return - ptx_thread_info *the_thread = (ptx_thread_info *)thd; - assert(the_thread != NULL); - return the_thread->get_return_PC(); -} - -address_type cuda_sim::get_converge_point(address_type pc) { - // the branch could encode the reconvergence point and/or a bit that indicates - // the - // reconvergence point is the return PC on the call stack in the case the - // branch has - // no immediate postdominator in the function (i.e., due to multiple return - // points). - - std::map::iterator f = g_pc_to_finfo.find(pc); - assert(f != g_pc_to_finfo.end()); - function_info *finfo = f->second; - rec_pts tmp = find_reconvergence_points(finfo); - - int i = 0; - for (; i < tmp.s_num_recon; ++i) { - if (tmp.s_kernel_recon_points[i].source_pc == pc) { - if (tmp.s_kernel_recon_points[i].target_pc == (unsigned)-2) { - return RECONVERGE_RETURN_PC; - } else { - return tmp.s_kernel_recon_points[i].target_pc; +void ptxinfo_opencl_addinfo( std::map &kernels ) +{ + + if(! g_ptxinfo_kname) { + printf ("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", + g_ptxinfo.gmem, + g_ptxinfo.cmem); + clear_ptxinfo(); + return; + } + + if( !strcmp("__cuda_dummy_entry__",g_ptxinfo_kname) ) { + // this string produced by ptxas for empty ptx files (e.g., bandwidth test) + clear_ptxinfo(); + return; + } + std::map::iterator k=kernels.find(g_ptxinfo_kname); + if( k==kernels.end() ) { + printf ("GPGPU-Sim PTX: ERROR ** implementation for '%s' not found.\n", g_ptxinfo_kname ); + abort(); + } else { + printf ("GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", + g_ptxinfo_kname, + g_ptxinfo.regs, + g_ptxinfo.lmem, + g_ptxinfo.smem, + g_ptxinfo.cmem ); + function_info *finfo = k->second; + assert(finfo!=NULL); + finfo->set_kernel_info( g_ptxinfo ); + } + clear_ptxinfo(); +} + +struct rec_pts cuda_sim::find_reconvergence_points( function_info *finfo ) +{ + rec_pts tmp; + std::map::iterator r=g_rpts.find(finfo); + + if( r==g_rpts.end() ) { + int num_recon = finfo->get_num_reconvergence_pairs(); + + gpgpu_recon_t *kernel_recon_points = (struct gpgpu_recon_t*) calloc(num_recon, sizeof(struct gpgpu_recon_t)); + finfo->get_reconvergence_pairs(kernel_recon_points); + printf("GPGPU-Sim PTX: reconvergence points for %s...\n", finfo->get_name().c_str() ); + for (int i=0;iprint_insn(); + printf("\n"); + printf("GPGPU-Sim PTX: immediate post dominator @ " ); + if( kernel_recon_points[i].target_inst ) + kernel_recon_points[i].target_inst->print_insn(); + printf("\n"); } - } - } - return NO_BRANCH_DIVERGENCE; + printf("GPGPU-Sim PTX: ... end of reconvergence points for %s\n", finfo->get_name().c_str() ); + + tmp.s_kernel_recon_points = kernel_recon_points; + tmp.s_num_recon = num_recon; + g_rpts[finfo] = tmp; + } else { + tmp = r->second; + } + return tmp; +} + +address_type get_return_pc( void *thd ) +{ + // function call return + ptx_thread_info *the_thread = (ptx_thread_info*)thd; + assert( the_thread != NULL ); + return the_thread->get_return_PC(); +} + +address_type cuda_sim::get_converge_point( address_type pc ) +{ + // the branch could encode the reconvergence point and/or a bit that indicates the + // reconvergence point is the return PC on the call stack in the case the branch has + // no immediate postdominator in the function (i.e., due to multiple return points). + + std::map::iterator f=g_pc_to_finfo.find(pc); + assert( f != g_pc_to_finfo.end() ); + function_info *finfo = f->second; + rec_pts tmp = find_reconvergence_points(finfo); + + int i=0; + for (; i < tmp.s_num_recon; ++i) { + if (tmp.s_kernel_recon_points[i].source_pc == pc) { + if( tmp.s_kernel_recon_points[i].target_pc == (unsigned) -2 ) { + return RECONVERGE_RETURN_PC; + } else { + return tmp.s_kernel_recon_points[i].target_pc; + } + } + } + return NO_BRANCH_DIVERGENCE; } -void functionalCoreSim::warp_exit(unsigned warp_id) { - for (int i = 0; i < m_warp_count * m_warp_size; i++) { - if (m_thread[i] != NULL) { - m_thread[i]->m_cta_info->register_deleted_thread(m_thread[i]); - delete m_thread[i]; +void functionalCoreSim::warp_exit( unsigned warp_id ) +{ + for(int i=0;im_cta_info->register_deleted_thread(m_thread[i]); + delete m_thread[i]; + } } - } } diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h index b828749..1be3d19 100644 --- a/src/cuda-sim/cuda-sim.h +++ b/src/cuda-sim/cuda-sim.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -30,13 +28,13 @@ #ifndef CUDASIM_H_INCLUDED #define CUDASIM_H_INCLUDED +#include "../abstract_hardware_model.h" +#include"../gpgpu-sim/shader.h" #include #include -#include #include -#include "../abstract_hardware_model.h" -#include "../gpgpu-sim/shader.h" -#include "ptx_sim.h" +#include +#include"ptx_sim.h" class gpgpu_context; class memory_space; @@ -46,65 +44,69 @@ class symbol_table; extern const char *g_gpgpusim_version_string; extern int g_debug_execution; -extern void print_splash(); +extern void print_splash(); -extern void ptxinfo_opencl_addinfo( - std::map &kernels); -unsigned ptx_sim_init_thread(kernel_info_t &kernel, - class ptx_thread_info **thread_info, int sid, - unsigned tid, unsigned threads_left, - unsigned num_threads, class core_t *core, - unsigned hw_cta_id, unsigned hw_warp_id, - gpgpu_t *gpu, - bool functionalSimulationMode = false); -const struct gpgpu_ptx_sim_info *ptx_sim_kernel_info( - const class function_info *kernel); +extern void ptxinfo_opencl_addinfo( std::map &kernels ); +unsigned ptx_sim_init_thread( kernel_info_t &kernel, + class ptx_thread_info** thread_info, + int sid, + unsigned tid, + unsigned threads_left, + unsigned num_threads, + class core_t *core, + unsigned hw_cta_id, + unsigned hw_warp_id, + gpgpu_t *gpu, + bool functionalSimulationMode = false); +const struct gpgpu_ptx_sim_info* ptx_sim_kernel_info(const class function_info *kernel); /*! - * This class functionally executes a kernel. It uses the basic data structures - * and procedures in core_t + * This class functionally executes a kernel. It uses the basic data structures and procedures in core_t */ -class functionalCoreSim : public core_t { - public: - functionalCoreSim(kernel_info_t *kernel, gpgpu_sim *g, unsigned warp_size) - : core_t(g, kernel, warp_size, kernel->threads_per_cta()) { - m_warpAtBarrier = new bool[m_warp_count]; - m_liveThreadCount = new unsigned[m_warp_count]; - } - virtual ~functionalCoreSim() { - warp_exit(0); - delete[] m_liveThreadCount; - delete[] m_warpAtBarrier; - } - //! executes all warps till completion - void execute(int inst_count, unsigned ctaid_cp); - virtual void warp_exit(unsigned warp_id); - virtual bool warp_waiting_at_barrier(unsigned warp_id) const { - return (m_warpAtBarrier[warp_id] || !(m_liveThreadCount[warp_id] > 0)); - } - - private: - void executeWarp(unsigned, bool &, bool &); - // initializes threads in the CTA block which we are executing - void initializeCTA(unsigned ctaid_cp); - virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, - unsigned tid) { - if (m_thread[tid] == NULL || m_thread[tid]->is_done()) { - m_liveThreadCount[tid / m_warp_size]--; +class functionalCoreSim: public core_t +{ +public: + functionalCoreSim(kernel_info_t * kernel, gpgpu_sim *g, unsigned warp_size) + : core_t( g, kernel, warp_size, kernel->threads_per_cta() ) + { + m_warpAtBarrier = new bool [m_warp_count]; + m_liveThreadCount = new unsigned [m_warp_count]; } - } - - // lunches the stack and set the threads count - void createWarp(unsigned warpId); - - // each warp live thread count and barrier indicator - unsigned *m_liveThreadCount; - bool *m_warpAtBarrier; + virtual ~functionalCoreSim(){ + warp_exit(0); + delete[] m_liveThreadCount; + delete[] m_warpAtBarrier; + } + //! executes all warps till completion + void execute(int inst_count, unsigned ctaid_cp); + virtual void warp_exit( unsigned warp_id ); + virtual bool warp_waiting_at_barrier( unsigned warp_id ) const + { + return (m_warpAtBarrier[warp_id] || !(m_liveThreadCount[warp_id]>0)); + } + +private: + void executeWarp(unsigned, bool &, bool &); + //initializes threads in the CTA block which we are executing + void initializeCTA(unsigned ctaid_cp); + virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid) + { + if(m_thread[tid]==NULL || m_thread[tid]->is_done()){ + m_liveThreadCount[tid/m_warp_size]--; + } + } + + // lunches the stack and set the threads count + void createWarp(unsigned warpId); + + //each warp live thread count and barrier indicator + unsigned * m_liveThreadCount; + bool* m_warpAtBarrier; }; #define RECONVERGE_RETURN_PC ((address_type)-2) #define NO_BRANCH_DIVERGENCE ((address_type)-1) -address_type get_return_pc(void *thd); +address_type get_return_pc( void *thd ); const char *get_ptxinfo_kname(); void print_ptxinfo(); void clear_ptxinfo(); @@ -112,98 +114,89 @@ struct gpgpu_ptx_sim_info get_ptxinfo(); class gpgpu_recon_t; struct rec_pts { - gpgpu_recon_t *s_kernel_recon_points; - int s_num_recon; + gpgpu_recon_t *s_kernel_recon_points; + int s_num_recon; }; + class cuda_sim { - public: - cuda_sim(gpgpu_context *ctx) { - g_ptx_sim_num_insn = 0; - g_ptx_kernel_count = - -1; // used for classification stat collection purposes - gpgpu_param_num_shaders = 0; - g_cuda_launch_blocking = false; - g_inst_classification_stat = NULL; - g_inst_op_classification_stat = NULL; - g_assemble_code_next_pc = 0; - g_debug_thread_uid = 0; - g_override_embedded_ptx = false; - ptx_tex_regs = NULL; - g_ptx_thread_info_delete_count = 0; - g_ptx_thread_info_uid_next = 1; - g_debug_pc = 0xBEEF1518; - gpgpu_ctx = ctx; - } - // global variables - char *opcode_latency_int; - char *opcode_latency_fp; - char *opcode_latency_dp; - char *opcode_latency_sfu; - char *opcode_latency_tensor; - char *opcode_initiation_int; - char *opcode_initiation_fp; - char *opcode_initiation_dp; - char *opcode_initiation_sfu; - char *opcode_initiation_tensor; - int cp_count; - int cp_cta_resume; - int g_ptxinfo_error_detected; - unsigned g_ptx_sim_num_insn; - char *cdp_latency_str; - int g_ptx_kernel_count; // used for classification stat collection purposes - std::map - g_global_name_lookup; // indexed by hostVar - std::map - g_const_name_lookup; // indexed by hostVar - int g_ptx_sim_mode; // if non-zero run functional simulation only (i.e., no - // notion of a clock cycle) - unsigned gpgpu_param_num_shaders; - class std::map g_rpts; - bool g_cuda_launch_blocking; - void **g_inst_classification_stat; - void **g_inst_op_classification_stat; - std::set g_globals; - std::set g_constants; - std::map g_pc_to_finfo; - int gpgpu_ptx_instruction_classification; - unsigned cdp_latency[5]; - unsigned g_assemble_code_next_pc; - int g_debug_thread_uid; - bool g_override_embedded_ptx; - std::set g_ptx_cta_info_sm_idx_used; - ptx_reg_t *ptx_tex_regs; - unsigned g_ptx_thread_info_delete_count; - unsigned g_ptx_thread_info_uid_next; - addr_t g_debug_pc; - // backward pointer - class gpgpu_context *gpgpu_ctx; - // global functions - void ptx_opcocde_latency_options(option_parser_t opp); - void gpgpu_cuda_ptx_sim_main_func(kernel_info_t &kernel, bool openCL = false); - int gpgpu_opencl_ptx_sim_main_func(kernel_info_t *grid); - void init_inst_classification_stat(); - kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, - gpgpu_ptx_sim_arg_list_t args, - struct dim3 gridDim, - struct dim3 blockDim, - gpgpu_t *gpu); - void gpgpu_ptx_sim_register_global_variable(void *hostVar, - const char *deviceName, - size_t size); - void gpgpu_ptx_sim_register_const_variable(void *, const char *deviceName, - size_t size); - void read_sim_environment_variables(); - void set_param_gpgpu_num_shaders(int num_shaders); - struct rec_pts find_reconvergence_points(function_info *finfo); - address_type get_converge_point(address_type pc); - void gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, - size_t count, size_t offset, int to, - gpgpu_t *gpu); - void ptx_print_insn(address_type pc, FILE *fp); - std::string ptx_get_insn_str(address_type pc); - template - bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc); + public: + cuda_sim( gpgpu_context* ctx ) { + g_ptx_sim_num_insn = 0; + g_ptx_kernel_count = -1; // used for classification stat collection purposes + gpgpu_param_num_shaders = 0; + g_cuda_launch_blocking = false; + g_inst_classification_stat = NULL; + g_inst_op_classification_stat= NULL; + g_assemble_code_next_pc=0; + g_debug_thread_uid = 0; + g_override_embedded_ptx = false; + ptx_tex_regs = NULL; + g_ptx_thread_info_delete_count=0; + g_ptx_thread_info_uid_next=1; + g_debug_pc = 0xBEEF1518; + gpgpu_ctx = ctx; + } + //global variables + char *opcode_latency_int; + char *opcode_latency_fp; + char *opcode_latency_dp; + char *opcode_latency_sfu; + char *opcode_latency_tensor; + char *opcode_initiation_int; + char *opcode_initiation_fp; + char *opcode_initiation_dp; + char *opcode_initiation_sfu; + char *opcode_initiation_tensor; + int cp_count; + int cp_cta_resume; + int g_ptxinfo_error_detected; + unsigned g_ptx_sim_num_insn; + char *cdp_latency_str; + int g_ptx_kernel_count; // used for classification stat collection purposes + std::map g_global_name_lookup; // indexed by hostVar + std::map g_const_name_lookup; // indexed by hostVar + int g_ptx_sim_mode; // if non-zero run functional simulation only (i.e., no notion of a clock cycle) + unsigned gpgpu_param_num_shaders; + class std::map g_rpts; + bool g_cuda_launch_blocking; + void ** g_inst_classification_stat; + void ** g_inst_op_classification_stat; + std::set g_globals; + std::set g_constants; + std::map g_pc_to_finfo; + int gpgpu_ptx_instruction_classification; + unsigned cdp_latency[5]; + unsigned g_assemble_code_next_pc; + int g_debug_thread_uid; + bool g_override_embedded_ptx; + std::set g_ptx_cta_info_sm_idx_used; + ptx_reg_t* ptx_tex_regs; + unsigned g_ptx_thread_info_delete_count; + unsigned g_ptx_thread_info_uid_next; + addr_t g_debug_pc; + // backward pointer + class gpgpu_context* gpgpu_ctx; + //global functions + void ptx_opcocde_latency_options (option_parser_t opp); + void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL = false ); + int gpgpu_opencl_ptx_sim_main_func( kernel_info_t *grid ); + void init_inst_classification_stat(); + kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, + gpgpu_ptx_sim_arg_list_t args, + struct dim3 gridDim, + struct dim3 blockDim, + gpgpu_t *gpu ); + void gpgpu_ptx_sim_register_global_variable(void *hostVar, const char *deviceName, size_t size ); + void gpgpu_ptx_sim_register_const_variable(void*, const char *deviceName, size_t size ); + void read_sim_environment_variables(); + void set_param_gpgpu_num_shaders(int num_shaders); + struct rec_pts find_reconvergence_points( function_info *finfo ); + address_type get_converge_point( address_type pc ); + void gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to, gpgpu_t *gpu ); + void ptx_print_insn( address_type pc, FILE *fp ); + std::string ptx_get_insn_str( address_type pc ); + template bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc); }; #endif diff --git a/src/cuda-sim/cuda_device_printf.cc b/src/cuda-sim/cuda_device_printf.cc index 5088a88..9ac0727 100644 --- a/src/cuda-sim/cuda_device_printf.cc +++ b/src/cuda-sim/cuda_device_printf.cc @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -30,90 +28,87 @@ #include "cuda_device_printf.h" #include "ptx_ir.h" -void decode_space(memory_space_t &space, ptx_thread_info *thread, - const operand_info &op, memory_space *&mem, addr_t &addr); +void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand_info &op, memory_space *&mem, addr_t &addr); -void my_cuda_printf(const char *fmtstr, const char *arg_list) { - FILE *fp = stdout; - unsigned i = 0, j = 0; - unsigned arg_offset = 0; - char buf[64]; - bool in_fmt = false; - while (fmtstr[i]) { - char c = fmtstr[i++]; - if (!in_fmt) { - if (c != '%') { - fprintf(fp, "%c", c); +void my_cuda_printf(const char *fmtstr,const char *arg_list) +{ + FILE *fp = stdout; + unsigned i=0,j=0; + unsigned arg_offset=0; + char buf[64]; + bool in_fmt=false; + while( fmtstr[i] ) { + char c = fmtstr[i++]; + if( !in_fmt ) { + if( c != '%' ) { + fprintf(fp,"%c",c); + } else { + in_fmt=true; + buf[0] = c; + j=1; + } } else { - in_fmt = true; - buf[0] = c; - j = 1; + if(!( c == 'u' || c == 'f' || c == 'd' )) { + printf("GPGPU-Sim PTX: ERROR ** printf parsing support is limited to %%u, %%f, %%d at present"); + abort(); + } + buf[j] = c; + buf[j+1] = 0; + void* ptr = (void*)&arg_list[arg_offset]; + //unsigned long long value = ((unsigned long long*)arg_list)[arg_offset]; + if( c == 'u' || c == 'd' ) { + fprintf(fp,buf,*((unsigned long long*)ptr)); + } else if( c == 'f' ) { + double tmp = *((double*)ptr); + fprintf(fp,buf,tmp); + } + arg_offset++; + in_fmt=false; } - } else { - if (!(c == 'u' || c == 'f' || c == 'd')) { - printf( - "GPGPU-Sim PTX: ERROR ** printf parsing support is limited to %%u, " - "%%f, %%d at present"); - abort(); - } - buf[j] = c; - buf[j + 1] = 0; - void *ptr = (void *)&arg_list[arg_offset]; - // unsigned long long value = ((unsigned long long*)arg_list)[arg_offset]; - if (c == 'u' || c == 'd') { - fprintf(fp, buf, *((unsigned long long *)ptr)); - } else if (c == 'f') { - double tmp = *((double *)ptr); - fprintf(fp, buf, tmp); - } - arg_offset++; - in_fmt = false; - } - } + } } -void gpgpusim_cuda_vprintf(const ptx_instruction *pI, ptx_thread_info *thread, - const function_info *target_func) { - char *fmtstr = NULL; - char *arg_list = NULL; - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - assert(n_args == 2); - for (unsigned arg = 0; arg < n_args; arg++) { - const operand_info &actual_param_op = - pI->operand_lookup(n_return + 1 + arg); - const symbol *formal_param = target_func->get_arg(arg); - unsigned size = formal_param->get_size_in_bytes(); - assert(formal_param->is_param_local()); - assert(actual_param_op.is_param_local()); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - unsigned long long buffer[1024]; - assert(size < 1024 * sizeof(unsigned long long)); - thread->m_local_mem->read(from_addr, size, buffer); - addr_t addr = - (addr_t)buffer[0]; // should be pointer to generic memory location - memory_space *mem = NULL; - memory_space_t space = generic_space; - decode_space(space, thread, actual_param_op, mem, - addr); // figure out which space - if (arg == 0) { - unsigned len = 0; - char b = 0; - do { // figure out length - mem->read(addr + len, 1, &b); - len++; - } while (b); - fmtstr = (char *)malloc(len + 64); - for (int i = 0; i < len; i++) mem->read(addr + i, 1, fmtstr + i); - // mem->read(addr,len,fmtstr); - } else { - unsigned len = thread->get_finfo()->local_mem_framesize(); - arg_list = (char *)malloc(len + 64); - for (int i = 0; i < len; i++) mem->read(addr + i, 1, arg_list + i); - // mem->read(addr,len,arg_list); - } - } - my_cuda_printf(fmtstr, arg_list); - free(fmtstr); - free(arg_list); +void gpgpusim_cuda_vprintf(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func ) +{ + char *fmtstr = NULL; + char *arg_list = NULL; + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + assert( n_args == 2 ); + for( unsigned arg=0; arg < n_args; arg ++ ) { + const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); + const symbol *formal_param = target_func->get_arg(arg); + unsigned size=formal_param->get_size_in_bytes(); + assert( formal_param->is_param_local() ); + assert( actual_param_op.is_param_local() ); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + unsigned long long buffer[1024]; + assert(size<1024*sizeof(unsigned long long)); + thread->m_local_mem->read(from_addr,size,buffer); + addr_t addr = (addr_t)buffer[0]; // should be pointer to generic memory location + memory_space *mem=NULL; + memory_space_t space = generic_space; + decode_space(space,thread,actual_param_op,mem,addr); // figure out which space + if( arg == 0 ) { + unsigned len = 0; + char b = 0; + do { // figure out length + mem->read(addr+len,1,&b); + len++; + } while(b); + fmtstr = (char*)malloc(len+64); + for( int i=0; i < len; i++ ) + mem->read(addr+i,1,fmtstr+i); + //mem->read(addr,len,fmtstr); + } else { + unsigned len = thread->get_finfo()->local_mem_framesize(); + arg_list = (char*)malloc(len+64); + for( int i=0; i < len; i++ ) + mem->read(addr+i,1,arg_list+i); + //mem->read(addr,len,arg_list); + } + } + my_cuda_printf(fmtstr,arg_list); + free(fmtstr); + free(arg_list); } diff --git a/src/cuda-sim/cuda_device_printf.h b/src/cuda-sim/cuda_device_printf.h index 20744bb..4e9baaa 100644 --- a/src/cuda-sim/cuda_device_printf.h +++ b/src/cuda-sim/cuda_device_printf.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -30,8 +28,6 @@ #ifndef CUDA_DEVICE_PRINTF_INCLUDED #define CUDA_DEVICE_PRINTF_INCLUDED -void gpgpusim_cuda_vprintf(const class ptx_instruction* pI, - class ptx_thread_info* thread, - const class function_info* target_func); +void gpgpusim_cuda_vprintf(const class ptx_instruction * pI, class ptx_thread_info * thread, const class function_info * target_func ); #endif diff --git a/src/cuda-sim/cuda_device_runtime.cc b/src/cuda-sim/cuda_device_runtime.cc index 257dd50..4baced5 100644 --- a/src/cuda-sim/cuda_device_runtime.cc +++ b/src/cuda-sim/cuda_device_runtime.cc @@ -1,329 +1,297 @@ -// Jin: cuda_device_runtime.cc -// Defines CUDA device runtime APIs for CDP support +//Jin: cuda_device_runtime.cc +//Defines CUDA device runtime APIs for CDP support + #include #include + + #if (CUDART_VERSION >= 5000) #define __CUDA_RUNTIME_API_H__ #include #include -#include "../../libcuda/gpgpu_context.h" #include "../gpgpu-sim/gpu-sim.h" -#include "../gpgpusim_entrypoint.h" -#include "../stream_manager.h" #include "cuda-sim.h" -#include "cuda_device_runtime.h" #include "ptx_ir.h" +#include "../stream_manager.h" +#include "../gpgpusim_entrypoint.h" +#include "cuda_device_runtime.h" +#include "../../libcuda/gpgpu_context.h" -#define DEV_RUNTIME_REPORT(a) \ - if (g_debug_execution) { \ - std::cout << __FILE__ << ", " << __LINE__ << ": " << a << "\n"; \ - std::cout.flush(); \ - } - -// Handling device runtime api: -// void * cudaGetParameterBufferV2(void *func, dim3 gridDimension, dim3 -// blockDimension, unsigned int sharedMemSize) -void cuda_device_runtime::gpgpusim_cuda_getParameterBufferV2( - const ptx_instruction *pI, ptx_thread_info *thread, - const function_info *target_func) { - DEV_RUNTIME_REPORT("Calling cudaGetParameterBufferV2"); - - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert(n_args == 4); - - function_info *child_kernel_entry; - struct dim3 grid_dim, block_dim; - unsigned int shared_mem; - - for (unsigned arg = 0; arg < n_args; arg++) { - const operand_info &actual_param_op = - pI->operand_lookup(n_return + 1 + arg); // param# - const symbol *formal_param = - target_func->get_arg(arg); // cudaGetParameterBufferV2_param_# - unsigned size = formal_param->get_size_in_bytes(); - assert(formal_param->is_param_local()); - assert(actual_param_op.is_param_local()); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - - if (arg == 0) { // function_info* for the child kernel - unsigned long long buf; - assert(size == sizeof(function_info *)); - thread->m_local_mem->read(from_addr, size, &buf); - child_kernel_entry = (function_info *)buf; - assert(child_kernel_entry); - DEV_RUNTIME_REPORT("child kernel name " - << child_kernel_entry->get_name()); - } else if (arg == 1) { // dim3 grid_dim for the child kernel - assert(size == sizeof(struct dim3)); - thread->m_local_mem->read(from_addr, size, &grid_dim); - DEV_RUNTIME_REPORT("grid (" << grid_dim.x << ", " << grid_dim.y << ", " - << grid_dim.z << ")"); - } else if (arg == 2) { // dim3 block_dim for the child kernel - assert(size == sizeof(struct dim3)); - thread->m_local_mem->read(from_addr, size, &block_dim); - DEV_RUNTIME_REPORT("block (" << block_dim.x << ", " << block_dim.y << ", " - << block_dim.z << ")"); - } else if (arg == 3) { // unsigned int shared_mem - assert(size == sizeof(unsigned int)); - thread->m_local_mem->read(from_addr, size, &shared_mem); - DEV_RUNTIME_REPORT("shared memory " << shared_mem); +#define DEV_RUNTIME_REPORT(a) \ + if( g_debug_execution ) { \ + std::cout << __FILE__ << ", " << __LINE__ << ": " << a << "\n"; \ + std::cout.flush(); \ + } + + + +//Handling device runtime api: +//void * cudaGetParameterBufferV2(void *func, dim3 gridDimension, dim3 blockDimension, unsigned int sharedMemSize) +void cuda_device_runtime::gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) +{ + DEV_RUNTIME_REPORT("Calling cudaGetParameterBufferV2"); + + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert( n_args == 4 ); + + function_info * child_kernel_entry; + struct dim3 grid_dim, block_dim; + unsigned int shared_mem; + + for( unsigned arg=0; arg < n_args; arg ++ ) { + const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# + const symbol *formal_param = target_func->get_arg(arg); //cudaGetParameterBufferV2_param_# + unsigned size=formal_param->get_size_in_bytes(); + assert( formal_param->is_param_local() ); + assert( actual_param_op.is_param_local() ); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + + if(arg == 0) {//function_info* for the child kernel + unsigned long long buf; + assert(size == sizeof(function_info *)); + thread->m_local_mem->read(from_addr, size, &buf); + child_kernel_entry = (function_info *)buf; + assert(child_kernel_entry); + DEV_RUNTIME_REPORT("child kernel name " << child_kernel_entry->get_name()); + } + else if(arg == 1) { //dim3 grid_dim for the child kernel + assert(size == sizeof(struct dim3)); + thread->m_local_mem->read(from_addr, size, & grid_dim); + DEV_RUNTIME_REPORT("grid (" << grid_dim.x << ", " << grid_dim.y << ", " << grid_dim.z << ")"); + } + else if(arg == 2) { //dim3 block_dim for the child kernel + assert(size == sizeof(struct dim3)); + thread->m_local_mem->read(from_addr, size, & block_dim); + DEV_RUNTIME_REPORT("block (" << block_dim.x << ", " << block_dim.y << ", " << block_dim.z << ")"); + } + else if(arg == 3) { //unsigned int shared_mem + assert(size == sizeof(unsigned int)); + thread->m_local_mem->read(from_addr, size, & shared_mem); + DEV_RUNTIME_REPORT("shared memory " << shared_mem); + } } - } - - // get total child kernel argument size and malloc buffer in global memory - unsigned child_kernel_arg_size = child_kernel_entry->get_args_aligned_size(); - void *param_buffer = thread->get_gpu()->gpu_malloc(child_kernel_arg_size); - g_total_param_size += ((child_kernel_arg_size + 255) / 256 * 256); - DEV_RUNTIME_REPORT("child kernel arg size total " - << child_kernel_arg_size - << ", parameter buffer allocated at " << param_buffer); - if (g_total_param_size > g_max_total_param_size) - g_max_total_param_size = g_total_param_size; - - // store param buffer address and launch config - device_launch_config_t device_launch_config(grid_dim, block_dim, shared_mem, - child_kernel_entry); - assert(g_cuda_device_launch_param_map.find(param_buffer) == - g_cuda_device_launch_param_map.end()); - g_cuda_device_launch_param_map[param_buffer] = device_launch_config; - - // copy the buffer address to retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 - const symbol *formal_return = target_func->get_return_var(); // void * - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaGetParameterBufferV2 return value has size of " - << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && - return_size == sizeof(void *)); - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, ¶m_buffer, NULL, - NULL); + + //get total child kernel argument size and malloc buffer in global memory + unsigned child_kernel_arg_size = child_kernel_entry->get_args_aligned_size(); + void * param_buffer = thread->get_gpu()->gpu_malloc(child_kernel_arg_size); + g_total_param_size += ((child_kernel_arg_size + 255) / 256 * 256); + DEV_RUNTIME_REPORT("child kernel arg size total " << child_kernel_arg_size << ", parameter buffer allocated at " << param_buffer); + if(g_total_param_size > g_max_total_param_size) + g_max_total_param_size = g_total_param_size; + + //store param buffer address and launch config + device_launch_config_t device_launch_config(grid_dim, block_dim, shared_mem, child_kernel_entry); + assert(g_cuda_device_launch_param_map.find(param_buffer) == g_cuda_device_launch_param_map.end()); + g_cuda_device_launch_param_map[param_buffer] = device_launch_config; + + //copy the buffer address to retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 + const symbol *formal_return = target_func->get_return_var(); //void * + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaGetParameterBufferV2 return value has size of " << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && return_size == sizeof(void *)); + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, ¶m_buffer, NULL, NULL); + } -// Handling device runtime api: -// cudaError_t cudaLaunchDeviceV2(void *parameterBuffer, cudaStream_t stream) -void cuda_device_runtime::gpgpusim_cuda_launchDeviceV2( - const ptx_instruction *pI, ptx_thread_info *thread, - const function_info *target_func) { - DEV_RUNTIME_REPORT("Calling cudaLaunchDeviceV2"); - - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert(n_args == 2); - - kernel_info_t *device_grid = NULL; - function_info *device_kernel_entry = NULL; - void *parameter_buffer; - struct CUstream_st *child_stream; - device_launch_config_t config; - device_launch_operation_t device_launch_op; - - for (unsigned arg = 0; arg < n_args; arg++) { - const operand_info &actual_param_op = - pI->operand_lookup(n_return + 1 + arg); // param# - const symbol *formal_param = - target_func->get_arg(arg); // cudaLaunchDeviceV2_param_# - unsigned size = formal_param->get_size_in_bytes(); - assert(formal_param->is_param_local()); - assert(actual_param_op.is_param_local()); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - - if (arg == 0) { // paramter buffer for child kernel (in global memory) - // get parameter_buffer from the cudaLaunchDeviceV2_param0 - assert(size == sizeof(void *)); - thread->m_local_mem->read(from_addr, size, ¶meter_buffer); - assert((size_t)parameter_buffer >= GLOBAL_HEAP_START); - DEV_RUNTIME_REPORT("Parameter buffer locating at global memory " - << parameter_buffer); - - // get child grid info through parameter_buffer address - assert(g_cuda_device_launch_param_map.find(parameter_buffer) != - g_cuda_device_launch_param_map.end()); - config = g_cuda_device_launch_param_map[parameter_buffer]; - // device_grid = op.grid; - device_kernel_entry = config.entry; - DEV_RUNTIME_REPORT("find device kernel " - << device_kernel_entry->get_name()); - - // PDOM analysis is done for Parent kernel but not for child kernel. - if (device_kernel_entry->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", - device_kernel_entry->get_name().c_str()); - } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", - device_kernel_entry->get_name().c_str()); - /* - * Some of the instructions like printf() gives the gpgpusim the wrong - * impression that it is a function call. - * As printf() doesnt have a body like functions do, doing pdom analysis - * for printf() causes a crash. - */ - if (device_kernel_entry->get_function_size() > 0) - device_kernel_entry->do_pdom(); - device_kernel_entry->set_pdom(); - } - - // copy data in parameter_buffer to device kernel param memory - unsigned device_kernel_arg_size = - device_kernel_entry->get_args_aligned_size(); - DEV_RUNTIME_REPORT("device_kernel_arg_size " << device_kernel_arg_size); - memory_space *device_kernel_param_mem; - - // create child kernel_info_t and index it with parameter_buffer address - gpgpu_t *gpu = thread->get_gpu(); - device_grid = new kernel_info_t( - config.grid_dim, config.block_dim, device_kernel_entry, - gpu->getNameArrayMapping(), gpu->getNameInfoMapping()); - device_grid->launch_cycle = gpu->gpu_sim_cycle + gpu->gpu_tot_sim_cycle; - kernel_info_t &parent_grid = thread->get_kernel(); - DEV_RUNTIME_REPORT( - "child kernel launched by " - << parent_grid.name() << ", cta (" << thread->get_ctaid().x << ", " - << thread->get_ctaid().y << ", " << thread->get_ctaid().z - << "), thread (" << thread->get_tid().x << ", " << thread->get_tid().y - << ", " << thread->get_tid().z << ")"); - device_grid->set_parent(&parent_grid, thread->get_ctaid(), - thread->get_tid()); - device_launch_op = device_launch_operation_t(device_grid, NULL); - device_kernel_param_mem = device_grid->get_param_memory(); // kernel - // param - size_t param_start_address = 0; - // copy in word - for (unsigned n = 0; n < device_kernel_arg_size; n += 4) { - unsigned int oneword; - thread->get_gpu()->get_global_memory()->read( - (size_t)parameter_buffer + n, 4, &oneword); - device_kernel_param_mem->write(param_start_address + n, 4, &oneword, - NULL, NULL); - } - } else if (arg == 1) { // cudaStream for the child kernel - - assert(size == sizeof(cudaStream_t)); - thread->m_local_mem->read(from_addr, size, &child_stream); - - kernel_info_t &parent_kernel = thread->get_kernel(); - if (child_stream == 0) { // default stream on device for current CTA - child_stream = - parent_kernel.get_default_stream_cta(thread->get_ctaid()); - DEV_RUNTIME_REPORT("launching child kernel " - << device_grid->get_uid() - << " to default stream of the cta " - << child_stream->get_uid() << ": " << child_stream); - } else { - assert(parent_kernel.cta_has_stream(thread->get_ctaid(), child_stream)); - DEV_RUNTIME_REPORT("launching child kernel " - << device_grid->get_uid() << " to stream " - << child_stream->get_uid() << ": " << child_stream); - } - - device_launch_op.stream = child_stream; +//Handling device runtime api: +//cudaError_t cudaLaunchDeviceV2(void *parameterBuffer, cudaStream_t stream) +void cuda_device_runtime::gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) { + DEV_RUNTIME_REPORT("Calling cudaLaunchDeviceV2"); + + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert( n_args == 2 ); + + kernel_info_t * device_grid = NULL; + function_info * device_kernel_entry = NULL; + void * parameter_buffer; + struct CUstream_st * child_stream; + device_launch_config_t config; + device_launch_operation_t device_launch_op; + + for( unsigned arg=0; arg < n_args; arg ++ ) { + const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# + const symbol *formal_param = target_func->get_arg(arg); //cudaLaunchDeviceV2_param_# + unsigned size=formal_param->get_size_in_bytes(); + assert( formal_param->is_param_local() ); + assert( actual_param_op.is_param_local() ); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + + if(arg == 0) {//paramter buffer for child kernel (in global memory) + //get parameter_buffer from the cudaLaunchDeviceV2_param0 + assert(size == sizeof(void *)); + thread->m_local_mem->read(from_addr, size, ¶meter_buffer); + assert((size_t)parameter_buffer >= GLOBAL_HEAP_START); + DEV_RUNTIME_REPORT("Parameter buffer locating at global memory " << parameter_buffer); + + //get child grid info through parameter_buffer address + assert(g_cuda_device_launch_param_map.find(parameter_buffer) != g_cuda_device_launch_param_map.end()); + config = g_cuda_device_launch_param_map[parameter_buffer]; + //device_grid = op.grid; + device_kernel_entry = config.entry; + DEV_RUNTIME_REPORT("find device kernel " << device_kernel_entry->get_name()); + + //PDOM analysis is done for Parent kernel but not for child kernel. + if (device_kernel_entry->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", device_kernel_entry->get_name().c_str() ); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", device_kernel_entry->get_name().c_str() ); + /* + * Some of the instructions like printf() gives the gpgpusim the wrong impression that it is a function call. + * As printf() doesnt have a body like functions do, doing pdom analysis for printf() causes a crash. + */ + if (device_kernel_entry->get_function_size() >0) + device_kernel_entry->do_pdom(); + device_kernel_entry->set_pdom(); + } + + //copy data in parameter_buffer to device kernel param memory + unsigned device_kernel_arg_size = device_kernel_entry->get_args_aligned_size(); + DEV_RUNTIME_REPORT("device_kernel_arg_size " << device_kernel_arg_size); + memory_space *device_kernel_param_mem; + + //create child kernel_info_t and index it with parameter_buffer address + gpgpu_t* gpu=thread->get_gpu(); + device_grid = new kernel_info_t(config.grid_dim, config.block_dim, device_kernel_entry, gpu->getNameArrayMapping(), gpu->getNameInfoMapping()); + device_grid->launch_cycle = gpu->gpu_sim_cycle + gpu->gpu_tot_sim_cycle; + kernel_info_t & parent_grid = thread->get_kernel(); + DEV_RUNTIME_REPORT("child kernel launched by " << parent_grid.name() << ", cta (" << + thread->get_ctaid().x << ", " << thread->get_ctaid().y << ", " << thread->get_ctaid().z << + "), thread (" << thread->get_tid().x << ", " << thread->get_tid().y << ", " << thread->get_tid().z << + ")"); + device_grid->set_parent(&parent_grid, thread->get_ctaid(), thread->get_tid()); + device_launch_op = device_launch_operation_t(device_grid, NULL); + device_kernel_param_mem = device_grid->get_param_memory(); //kernel param + size_t param_start_address = 0; + //copy in word + for(unsigned n = 0; n < device_kernel_arg_size; n += 4) { + unsigned int oneword; + thread->get_gpu()->get_global_memory()->read((size_t)parameter_buffer + n, 4, &oneword); + device_kernel_param_mem->write(param_start_address + n, 4, &oneword, NULL, NULL); + } + } + else if(arg == 1) { //cudaStream for the child kernel + + assert(size == sizeof(cudaStream_t)); + thread->m_local_mem->read(from_addr, size, &child_stream); + + kernel_info_t & parent_kernel = thread->get_kernel(); + if(child_stream == 0) { //default stream on device for current CTA + child_stream = parent_kernel.get_default_stream_cta(thread->get_ctaid()); + DEV_RUNTIME_REPORT("launching child kernel " << device_grid->get_uid() << + " to default stream of the cta " << child_stream->get_uid() << ": " << child_stream); + } + else { + assert(parent_kernel.cta_has_stream(thread->get_ctaid(), child_stream)); + DEV_RUNTIME_REPORT("launching child kernel " << device_grid->get_uid() << + " to stream " << child_stream->get_uid() << ": " << child_stream); + } + + device_launch_op.stream = child_stream; + } + } - } - - // launch child kernel - g_cuda_device_launch_op.push_back(device_launch_op); - g_cuda_device_launch_param_map.erase(parameter_buffer); - - // set retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 - const symbol *formal_return = target_func->get_return_var(); // cudaError_t - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaLaunchDeviceV2 return value has size of " - << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && - return_size == sizeof(cudaError_t)); - cudaError_t error = cudaSuccess; - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); + + + //launch child kernel + g_cuda_device_launch_op.push_back(device_launch_op); + g_cuda_device_launch_param_map.erase(parameter_buffer); + + //set retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 + const symbol *formal_return = target_func->get_return_var(); //cudaError_t + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaLaunchDeviceV2 return value has size of " << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size + && return_size == sizeof(cudaError_t)); + cudaError_t error = cudaSuccess; + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); + } -// Handling device runtime api: -// cudaError_t cudaStreamCreateWithFlags ( cudaStream_t* pStream, unsigned int -// flags) -// flags can only be cudaStreamNonBlocking -void cuda_device_runtime::gpgpusim_cuda_streamCreateWithFlags( - const ptx_instruction *pI, ptx_thread_info *thread, - const function_info *target_func) { - DEV_RUNTIME_REPORT("Calling cudaStreamCreateWithFlags"); - - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert(n_args == 2); - - size_t generic_pStream_addr; - addr_t pStream_addr; - unsigned int flags; - for (unsigned arg = 0; arg < n_args; arg++) { - const operand_info &actual_param_op = - pI->operand_lookup(n_return + 1 + arg); // param# - const symbol *formal_param = - target_func->get_arg(arg); // cudaStreamCreateWithFlags_param_# - unsigned size = formal_param->get_size_in_bytes(); - assert(formal_param->is_param_local()); - assert(actual_param_op.is_param_local()); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - - if (arg == 0) { // cudaStream_t * pStream, address of cudaStream_t - assert(size == sizeof(cudaStream_t *)); - thread->m_local_mem->read(from_addr, size, &generic_pStream_addr); - - // pStream should be non-zero address in local memory - pStream_addr = generic_to_local( - thread->get_hw_sid(), thread->get_hw_tid(), generic_pStream_addr); - - DEV_RUNTIME_REPORT("pStream locating at local memory " << pStream_addr); - } else if (arg == - 1) { // unsigned int flags, should be cudaStreamNonBlocking - assert(size == sizeof(unsigned int)); - thread->m_local_mem->read(from_addr, size, &flags); - assert(flags == cudaStreamNonBlocking); + +//Handling device runtime api: +//cudaError_t cudaStreamCreateWithFlags ( cudaStream_t* pStream, unsigned int flags) +//flags can only be cudaStreamNonBlocking +void cuda_device_runtime::gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) { + DEV_RUNTIME_REPORT("Calling cudaStreamCreateWithFlags"); + + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert( n_args == 2 ); + + size_t generic_pStream_addr; + addr_t pStream_addr; + unsigned int flags; + for( unsigned arg=0; arg < n_args; arg ++ ) { + const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# + const symbol *formal_param = target_func->get_arg(arg); //cudaStreamCreateWithFlags_param_# + unsigned size=formal_param->get_size_in_bytes(); + assert( formal_param->is_param_local() ); + assert( actual_param_op.is_param_local() ); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + + if(arg == 0) {//cudaStream_t * pStream, address of cudaStream_t + assert(size == sizeof(cudaStream_t *)); + thread->m_local_mem->read(from_addr, size, &generic_pStream_addr); + + //pStream should be non-zero address in local memory + pStream_addr = generic_to_local(thread->get_hw_sid(), thread->get_hw_tid(), generic_pStream_addr); + + DEV_RUNTIME_REPORT("pStream locating at local memory " << pStream_addr); + } + else if(arg == 1) { //unsigned int flags, should be cudaStreamNonBlocking + assert(size == sizeof(unsigned int)); + thread->m_local_mem->read(from_addr, size, &flags); + assert(flags == cudaStreamNonBlocking); + } } - } - - // create stream and write back to param0 - CUstream_st *stream = - thread->get_kernel().create_stream_cta(thread->get_ctaid()); - DEV_RUNTIME_REPORT("Create stream " << stream->get_uid() << ": " << stream); - thread->m_local_mem->write(pStream_addr, sizeof(cudaStream_t), &stream, NULL, - NULL); - - // set retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 - const symbol *formal_return = target_func->get_return_var(); // cudaError_t - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaStreamCreateWithFlags return value has size of " - << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && - return_size == sizeof(cudaError_t)); - cudaError_t error = cudaSuccess; - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); + + //create stream and write back to param0 + CUstream_st * stream = thread->get_kernel().create_stream_cta(thread->get_ctaid()); + DEV_RUNTIME_REPORT("Create stream " << stream->get_uid() << ": " << stream); + thread->m_local_mem->write(pStream_addr, sizeof(cudaStream_t), &stream, NULL, NULL); + + //set retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 + const symbol *formal_return = target_func->get_return_var(); //cudaError_t + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaStreamCreateWithFlags return value has size of " << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size + && return_size == sizeof(cudaError_t)); + cudaError_t error = cudaSuccess; + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); + } + void cuda_device_runtime::launch_one_device_kernel() { - if (!g_cuda_device_launch_op.empty()) { - device_launch_operation_t &op = g_cuda_device_launch_op.front(); - - stream_operation stream_op = stream_operation( - op.grid, gpgpu_ctx->func_sim->g_ptx_sim_mode, op.stream); - gpgpu_ctx->the_gpgpusim->g_stream_manager->push(stream_op); - g_cuda_device_launch_op.pop_front(); - } + if(!g_cuda_device_launch_op.empty()) { + device_launch_operation_t &op = g_cuda_device_launch_op.front(); + + stream_operation stream_op = stream_operation(op.grid, gpgpu_ctx->func_sim->g_ptx_sim_mode, op.stream); + gpgpu_ctx->the_gpgpusim->g_stream_manager->push(stream_op); + g_cuda_device_launch_op.pop_front(); + } } void cuda_device_runtime::launch_all_device_kernels() { - while (!g_cuda_device_launch_op.empty()) { - launch_one_device_kernel(); - } + while(!g_cuda_device_launch_op.empty()) { + launch_one_device_kernel(); + } } #endif diff --git a/src/cuda-sim/cuda_device_runtime.h b/src/cuda-sim/cuda_device_runtime.h index 1d661b2..7f7a0ca 100644 --- a/src/cuda-sim/cuda_device_runtime.h +++ b/src/cuda-sim/cuda_device_runtime.h @@ -1,66 +1,67 @@ #ifndef __cuda_device_runtime_h__ #define __cuda_device_runtime_h__ -// Jin: cuda_device_runtime.h -// Defines CUDA device runtime APIs for CDP support +//Jin: cuda_device_runtime.h +//Defines CUDA device runtime APIs for CDP support class device_launch_config_t { - public: - device_launch_config_t() {} - device_launch_config_t(dim3 _grid_dim, dim3 _block_dim, - unsigned int _shared_mem, function_info* _entry) - : grid_dim(_grid_dim), - block_dim(_block_dim), - shared_mem(_shared_mem), - entry(_entry) {} +public: + device_launch_config_t() {} + + device_launch_config_t(dim3 _grid_dim, + dim3 _block_dim, + unsigned int _shared_mem, + function_info * _entry): + grid_dim(_grid_dim), + block_dim(_block_dim), + shared_mem(_shared_mem), + entry(_entry) {} + + dim3 grid_dim; + dim3 block_dim; + unsigned int shared_mem; + function_info * entry; - dim3 grid_dim; - dim3 block_dim; - unsigned int shared_mem; - function_info* entry; }; class device_launch_operation_t { - public: - device_launch_operation_t() {} - device_launch_operation_t(kernel_info_t* _grid, CUstream_st* _stream) - : grid(_grid), stream(_stream) {} - kernel_info_t* grid; // a new child grid +public: + device_launch_operation_t() {} + device_launch_operation_t(kernel_info_t *_grid, + CUstream_st * _stream) : + grid(_grid), stream(_stream) {} + + kernel_info_t * grid; //a new child grid + + CUstream_st * stream; - CUstream_st* stream; }; class gpgpu_context; class cuda_device_runtime { - public: - cuda_device_runtime(gpgpu_context* ctx) { - g_total_param_size = 0; - g_max_total_param_size = 0; - gpgpu_ctx = ctx; - } - unsigned long long g_total_param_size; - std::map g_cuda_device_launch_param_map; - std::list g_cuda_device_launch_op; - unsigned g_kernel_launch_latency; - unsigned long long g_max_total_param_size; - bool g_cdp_enabled; + public: + cuda_device_runtime( gpgpu_context* ctx ) { + g_total_param_size = 0; + g_max_total_param_size = 0; + gpgpu_ctx = ctx; + } + unsigned long long g_total_param_size; + std::map g_cuda_device_launch_param_map; + std::list g_cuda_device_launch_op; + unsigned g_kernel_launch_latency; + unsigned long long g_max_total_param_size; + bool g_cdp_enabled; - // backward pointer - class gpgpu_context* gpgpu_ctx; + // backward pointer + class gpgpu_context* gpgpu_ctx; #if (CUDART_VERSION >= 5000) #pragma once - void gpgpusim_cuda_launchDeviceV2(const ptx_instruction* pI, - ptx_thread_info* thread, - const function_info* target_func); - void gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction* pI, - ptx_thread_info* thread, - const function_info* target_func); - void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction* pI, - ptx_thread_info* thread, - const function_info* target_func); - void launch_all_device_kernels(); - void launch_one_device_kernel(); + void gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); + void gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); + void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); + void launch_all_device_kernels(); + void launch_one_device_kernel(); #endif }; diff --git a/src/cuda-sim/half.h b/src/cuda-sim/half.h index d33b03c..9f74bb7 100644 --- a/src/cuda-sim/half.h +++ b/src/cuda-sim/half.h @@ -2,25 +2,17 @@ // // Copyright (c) 2012-2017 Christian Rau // -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation -// files (the "Software"), to deal in the Software without restriction, -// including without limitation the rights to use, copy, -// modify, merge, publish, distribute, sublicense, and/or sell copies of the -// Software, and to permit persons to whom the +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, +// modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. +// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. // -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE -// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // Version 1.12.0 @@ -31,191 +23,180 @@ #define HALF_HALF_HPP /// Combined gcc version number. -#define HALF_GNUC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -// check C++11 language features -#if defined(__clang__) // clang -#if __has_feature(cxx_static_assert) && \ - !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) -#define HALF_ENABLE_CPP11_STATIC_ASSERT 1 -#endif -#if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR) -#define HALF_ENABLE_CPP11_CONSTEXPR 1 -#endif -#if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT) -#define HALF_ENABLE_CPP11_NOEXCEPT 1 -#endif -#if __has_feature(cxx_user_literals) && \ - !defined(HALF_ENABLE_CPP11_USER_LITERALS) -#define HALF_ENABLE_CPP11_USER_LITERALS 1 -#endif -#if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && \ - !defined(HALF_ENABLE_CPP11_LONG_LONG) -#define HALF_ENABLE_CPP11_LONG_LONG 1 -#endif -/*#elif defined(__INTEL_COMPILER) - //Intel C++ - #if __INTEL_COMPILER >= 1100 && - !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) ???????? - #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 - #endif - #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) - ???????? - #define HALF_ENABLE_CPP11_CONSTEXPR 1 - #endif - #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) - ???????? - #define HALF_ENABLE_CPP11_NOEXCEPT 1 - #endif - #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_LONG_LONG) - ???????? - #define HALF_ENABLE_CPP11_LONG_LONG 1 - #endif*/ -#elif defined(__GNUC__) // gcc -#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L -#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) -#define HALF_ENABLE_CPP11_STATIC_ASSERT 1 -#endif -#if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) -#define HALF_ENABLE_CPP11_CONSTEXPR 1 -#endif -#if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) -#define HALF_ENABLE_CPP11_NOEXCEPT 1 -#endif -#if HALF_GNUC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) -#define HALF_ENABLE_CPP11_USER_LITERALS 1 -#endif -#if !defined(HALF_ENABLE_CPP11_LONG_LONG) -#define HALF_ENABLE_CPP11_LONG_LONG 1 -#endif -#endif -#elif defined(_MSC_VER) // Visual C++ -#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) -#define HALF_ENABLE_CPP11_CONSTEXPR 1 -#endif -#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) -#define HALF_ENABLE_CPP11_NOEXCEPT 1 -#endif -#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) -#define HALF_ENABLE_CPP11_USER_LITERALS 1 -#endif -#if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) -#define HALF_ENABLE_CPP11_STATIC_ASSERT 1 -#endif -#if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG) -#define HALF_ENABLE_CPP11_LONG_LONG 1 -#endif -#define HALF_POP_WARNINGS 1 -#pragma warning(push) -#pragma warning(disable : 4099 4127 4146) // struct vs class, constant in if, - // negative unsigned -#endif - -// check C++11 library features +#define HALF_GNUC_VERSION (__GNUC__*100+__GNUC_MINOR__) + +//check C++11 language features +#if defined(__clang__) //clang + #if __has_feature(cxx_static_assert) && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) + #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 + #endif + #if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR) + #define HALF_ENABLE_CPP11_CONSTEXPR 1 + #endif + #if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT) + #define HALF_ENABLE_CPP11_NOEXCEPT 1 + #endif + #if __has_feature(cxx_user_literals) && !defined(HALF_ENABLE_CPP11_USER_LITERALS) + #define HALF_ENABLE_CPP11_USER_LITERALS 1 + #endif + #if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && !defined(HALF_ENABLE_CPP11_LONG_LONG) + #define HALF_ENABLE_CPP11_LONG_LONG 1 + #endif +/*#elif defined(__INTEL_COMPILER) //Intel C++ + #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) ???????? + #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 + #endif + #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) ???????? + #define HALF_ENABLE_CPP11_CONSTEXPR 1 + #endif + #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) ???????? + #define HALF_ENABLE_CPP11_NOEXCEPT 1 + #endif + #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_LONG_LONG) ???????? + #define HALF_ENABLE_CPP11_LONG_LONG 1 + #endif*/ +#elif defined(__GNUC__) //gcc + #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L + #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) + #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 + #endif + #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) + #define HALF_ENABLE_CPP11_CONSTEXPR 1 + #endif + #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) + #define HALF_ENABLE_CPP11_NOEXCEPT 1 + #endif + #if HALF_GNUC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) + #define HALF_ENABLE_CPP11_USER_LITERALS 1 + #endif + #if !defined(HALF_ENABLE_CPP11_LONG_LONG) + #define HALF_ENABLE_CPP11_LONG_LONG 1 + #endif + #endif +#elif defined(_MSC_VER) //Visual C++ + #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) + #define HALF_ENABLE_CPP11_CONSTEXPR 1 + #endif + #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) + #define HALF_ENABLE_CPP11_NOEXCEPT 1 + #endif + #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) + #define HALF_ENABLE_CPP11_USER_LITERALS 1 + #endif + #if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) + #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 + #endif + #if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG) + #define HALF_ENABLE_CPP11_LONG_LONG 1 + #endif + #define HALF_POP_WARNINGS 1 + #pragma warning(push) + #pragma warning(disable : 4099 4127 4146) //struct vs class, constant in if, negative unsigned +#endif + +//check C++11 library features #include -#if defined(_LIBCPP_VERSION) // libc++ -#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 -#ifndef HALF_ENABLE_CPP11_TYPE_TRAITS -#define HALF_ENABLE_CPP11_TYPE_TRAITS 1 -#endif -#ifndef HALF_ENABLE_CPP11_CSTDINT -#define HALF_ENABLE_CPP11_CSTDINT 1 -#endif -#ifndef HALF_ENABLE_CPP11_CMATH -#define HALF_ENABLE_CPP11_CMATH 1 -#endif -#ifndef HALF_ENABLE_CPP11_HASH -#define HALF_ENABLE_CPP11_HASH 1 -#endif -#endif -#elif defined(__GLIBCXX__) // libstdc++ -#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 -#ifdef __clang__ -#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) -#define HALF_ENABLE_CPP11_TYPE_TRAITS 1 -#endif -#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT) -#define HALF_ENABLE_CPP11_CSTDINT 1 -#endif -#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH) -#define HALF_ENABLE_CPP11_CMATH 1 -#endif -#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH) -#define HALF_ENABLE_CPP11_HASH 1 -#endif -#else -#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT) -#define HALF_ENABLE_CPP11_CSTDINT 1 -#endif -#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH) -#define HALF_ENABLE_CPP11_CMATH 1 -#endif -#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH) -#define HALF_ENABLE_CPP11_HASH 1 -#endif -#endif -#endif -#elif defined(_CPPLIB_VER) // Dinkumware/Visual C++ -#if _CPPLIB_VER >= 520 -#ifndef HALF_ENABLE_CPP11_TYPE_TRAITS -#define HALF_ENABLE_CPP11_TYPE_TRAITS 1 -#endif -#ifndef HALF_ENABLE_CPP11_CSTDINT -#define HALF_ENABLE_CPP11_CSTDINT 1 -#endif -#ifndef HALF_ENABLE_CPP11_HASH -#define HALF_ENABLE_CPP11_HASH 1 -#endif -#endif -#if _CPPLIB_VER >= 610 -#ifndef HALF_ENABLE_CPP11_CMATH -#define HALF_ENABLE_CPP11_CMATH 1 -#endif -#endif +#if defined(_LIBCPP_VERSION) //libc++ + #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 + #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS + #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 + #endif + #ifndef HALF_ENABLE_CPP11_CSTDINT + #define HALF_ENABLE_CPP11_CSTDINT 1 + #endif + #ifndef HALF_ENABLE_CPP11_CMATH + #define HALF_ENABLE_CPP11_CMATH 1 + #endif + #ifndef HALF_ENABLE_CPP11_HASH + #define HALF_ENABLE_CPP11_HASH 1 + #endif + #endif +#elif defined(__GLIBCXX__) //libstdc++ + #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 + #ifdef __clang__ + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) + #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 + #endif + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT) + #define HALF_ENABLE_CPP11_CSTDINT 1 + #endif + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH) + #define HALF_ENABLE_CPP11_CMATH 1 + #endif + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH) + #define HALF_ENABLE_CPP11_HASH 1 + #endif + #else + #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT) + #define HALF_ENABLE_CPP11_CSTDINT 1 + #endif + #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH) + #define HALF_ENABLE_CPP11_CMATH 1 + #endif + #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH) + #define HALF_ENABLE_CPP11_HASH 1 + #endif + #endif + #endif +#elif defined(_CPPLIB_VER) //Dinkumware/Visual C++ + #if _CPPLIB_VER >= 520 + #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS + #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 + #endif + #ifndef HALF_ENABLE_CPP11_CSTDINT + #define HALF_ENABLE_CPP11_CSTDINT 1 + #endif + #ifndef HALF_ENABLE_CPP11_HASH + #define HALF_ENABLE_CPP11_HASH 1 + #endif + #endif + #if _CPPLIB_VER >= 610 + #ifndef HALF_ENABLE_CPP11_CMATH + #define HALF_ENABLE_CPP11_CMATH 1 + #endif + #endif #endif #undef HALF_GNUC_VERSION -// support constexpr +//support constexpr #if HALF_ENABLE_CPP11_CONSTEXPR -#define HALF_CONSTEXPR constexpr -#define HALF_CONSTEXPR_CONST constexpr + #define HALF_CONSTEXPR constexpr + #define HALF_CONSTEXPR_CONST constexpr #else -#define HALF_CONSTEXPR -#define HALF_CONSTEXPR_CONST const + #define HALF_CONSTEXPR + #define HALF_CONSTEXPR_CONST const #endif -// support noexcept +//support noexcept #if HALF_ENABLE_CPP11_NOEXCEPT -#define HALF_NOEXCEPT noexcept -#define HALF_NOTHROW noexcept + #define HALF_NOEXCEPT noexcept + #define HALF_NOTHROW noexcept #else -#define HALF_NOEXCEPT -#define HALF_NOTHROW throw() + #define HALF_NOEXCEPT + #define HALF_NOTHROW throw() #endif #include +#include +#include #include #include #include -#include -#include #if HALF_ENABLE_CPP11_TYPE_TRAITS -#include + #include #endif #if HALF_ENABLE_CPP11_CSTDINT -#include + #include #endif #if HALF_ENABLE_CPP11_HASH -#include + #include #endif + /// Default rounding mode. -/// This specifies the rounding mode used for all conversions between -/// [half](\ref half_float::half)s and `float`s as well as -/// for the half_cast() if not specifying a rounding mode explicitly. It can be -/// redefined (before including half.hpp) to one -/// of the standard rounding modes using their respective constants or the -/// equivalent values of `std::float_round_style`: +/// This specifies the rounding mode used for all conversions between [half](\ref half_float::half)s and `float`s as well as +/// for the half_cast() if not specifying a rounding mode explicitly. It can be redefined (before including half.hpp) to one +/// of the standard rounding modes using their respective constants or the equivalent values of `std::float_round_style`: /// /// `std::float_round_style` | value | rounding /// ---------------------------------|-------|------------------------- @@ -225,354 +206,256 @@ /// `std::round_toward_infinity` | 2 | toward positive infinity /// `std::round_toward_neg_infinity` | 3 | toward negative infinity /// -/// By default this is set to `-1` (`std::round_indeterminate`), which uses -/// truncation (round toward zero, but with overflows -/// set to infinity) and is the fastest rounding mode possible. It can even be -/// set to `std::numeric_limits::round_style` -/// to synchronize the rounding mode with that of the underlying -/// single-precision implementation. +/// By default this is set to `-1` (`std::round_indeterminate`), which uses truncation (round toward zero, but with overflows +/// set to infinity) and is the fastest rounding mode possible. It can even be set to `std::numeric_limits::round_style` +/// to synchronize the rounding mode with that of the underlying single-precision implementation. #ifndef HALF_ROUND_STYLE -#define HALF_ROUND_STYLE -1 // = std::round_indeterminate + #define HALF_ROUND_STYLE -1 // = std::round_indeterminate #endif /// Tie-breaking behaviour for round to nearest. -/// This specifies if ties in round to nearest should be resolved by rounding to -/// the nearest even value. By default this is -/// defined to `0` resulting in the faster but slightly more biased behaviour of -/// rounding away from zero in half-way cases (and -/// thus equal to the round() function), but can be redefined to `1` (before -/// including half.hpp) if more IEEE-conformant +/// This specifies if ties in round to nearest should be resolved by rounding to the nearest even value. By default this is +/// defined to `0` resulting in the faster but slightly more biased behaviour of rounding away from zero in half-way cases (and +/// thus equal to the round() function), but can be redefined to `1` (before including half.hpp) if more IEEE-conformant /// behaviour is needed. #ifndef HALF_ROUND_TIES_TO_EVEN -#define HALF_ROUND_TIES_TO_EVEN 0 // ties away from zero + #define HALF_ROUND_TIES_TO_EVEN 0 // ties away from zero #endif /// Value signaling overflow. -/// In correspondence with `HUGE_VAL[F|L]` from `` this symbol expands to -/// a positive value signaling the overflow of an +/// In correspondence with `HUGE_VAL[F|L]` from `` this symbol expands to a positive value signaling the overflow of an /// operation, in particular it just evaluates to positive infinity. -#define HUGE_VALH std::numeric_limits::infinity() +#define HUGE_VALH std::numeric_limits::infinity() /// Fast half-precision fma function. -/// This symbol is only defined if the fma() function generally executes as fast -/// as, or faster than, a separate -/// half-precision multiplication followed by an addition. Due to the internal -/// single-precision implementation of all +/// This symbol is only defined if the fma() function generally executes as fast as, or faster than, a separate +/// half-precision multiplication followed by an addition. Due to the internal single-precision implementation of all /// arithmetic operations, this is in fact always the case. -#define FP_FAST_FMAH 1 +#define FP_FAST_FMAH 1 #ifndef FP_ILOGB0 -#define FP_ILOGB0 INT_MIN + #define FP_ILOGB0 INT_MIN #endif #ifndef FP_ILOGBNAN -#define FP_ILOGBNAN INT_MAX + #define FP_ILOGBNAN INT_MAX #endif #ifndef FP_SUBNORMAL -#define FP_SUBNORMAL 0 + #define FP_SUBNORMAL 0 #endif #ifndef FP_ZERO -#define FP_ZERO 1 + #define FP_ZERO 1 #endif #ifndef FP_NAN -#define FP_NAN 2 + #define FP_NAN 2 #endif #ifndef FP_INFINITE -#define FP_INFINITE 3 + #define FP_INFINITE 3 #endif #ifndef FP_NORMAL -#define FP_NORMAL 4 + #define FP_NORMAL 4 #endif + /// Main namespace for half precision functionality. /// This namespace contains all the functionality provided by the library. -namespace half_float { -class half; +namespace half_float +{ + class half; #if HALF_ENABLE_CPP11_USER_LITERALS -/// Library-defined half-precision literals. -/// Import this namespace to enable half-precision floating point literals: -/// ~~~~{.cpp} -/// using namespace half_float::literal; -/// half_float::half = 4.2_h; -/// ~~~~ -namespace literal { -half operator"" _h(long double); -} -#endif - -/// \internal -/// \brief Implementation details. -namespace detail { -#if HALF_ENABLE_CPP11_TYPE_TRAITS -/// Conditional type. -template -struct conditional : std::conditional {}; - -/// Helper for tag dispatching. -template -struct bool_type : std::integral_constant {}; -using std::true_type; -using std::false_type; - -/// Type traits for floating point types. -template -struct is_float : std::is_floating_point {}; -#else -/// Conditional type. -template -struct conditional { - typedef T type; -}; -template -struct conditional { - typedef F type; -}; - -/// Helper for tag dispatching. -template -struct bool_type {}; -typedef bool_type true_type; -typedef bool_type false_type; - -/// Type traits for floating point types. -template -struct is_float : false_type {}; -template -struct is_float : is_float {}; -template -struct is_float : is_float {}; -template -struct is_float : is_float {}; -template <> -struct is_float : true_type {}; -template <> -struct is_float : true_type {}; -template <> -struct is_float : true_type {}; -#endif - -/// Type traits for floating point bits. -template -struct bits { - typedef unsigned char type; -}; -template -struct bits : bits {}; -template -struct bits : bits {}; -template -struct bits : bits {}; - -#if HALF_ENABLE_CPP11_CSTDINT -/// Unsigned integer of (at least) 16 bits width. -typedef std::uint_least16_t uint16; - -/// Unsigned integer of (at least) 32 bits width. -template <> -struct bits { - typedef std::uint_least32_t type; -}; - -/// Unsigned integer of (at least) 64 bits width. -template <> -struct bits { - typedef std::uint_least64_t type; -}; -#else -/// Unsigned integer of (at least) 16 bits width. -typedef unsigned short uint16; - -/// Unsigned integer of (at least) 32 bits width. -template <> -struct bits - : conditional::digits >= 32, unsigned int, - unsigned long> {}; - -#if HALF_ENABLE_CPP11_LONG_LONG -/// Unsigned integer of (at least) 64 bits width. -template <> -struct bits - : conditional::digits >= 64, - unsigned long, unsigned long long> {}; -#else -/// Unsigned integer of (at least) 64 bits width. -template <> -struct bits { - typedef unsigned long type; -}; -#endif -#endif - -/// Tag type for binary construction. -struct binary_t {}; - -/// Tag for binary construction. -HALF_CONSTEXPR_CONST binary_t binary = binary_t(); - -/// Temporary half-precision expression. -/// This class represents a half-precision expression which just stores a -/// single-precision value internally. -struct expr { - /// Conversion constructor. - /// \param f single-precision value to convert - explicit HALF_CONSTEXPR expr(float f) HALF_NOEXCEPT : value_(f) {} - - /// Conversion to single-precision. - /// \return single precision value representing expression value - HALF_CONSTEXPR operator float() const HALF_NOEXCEPT { return value_; } - - private: - /// Internal expression value stored in single-precision. - float value_; -}; - -/// SFINAE helper for generic half-precision functions. -/// This class template has to be specialized for each valid combination of -/// argument types to provide a corresponding -/// `type` member equivalent to \a T. -/// \tparam T type to return -template -struct enable {}; -template -struct enable { - typedef T type; -}; -template -struct enable { - typedef T type; -}; -template -struct enable { - typedef T type; -}; -template -struct enable { - typedef T type; -}; -template -struct enable { - typedef T type; -}; -template -struct enable { - typedef T type; -}; -template -struct enable { - typedef T type; -}; -template -struct enable { - typedef T type; -}; -template -struct enable { - typedef T type; -}; -template -struct enable { - typedef T type; -}; -template -struct enable { - typedef T type; -}; -template -struct enable { - typedef T type; -}; -template -struct enable { - typedef T type; -}; -template -struct enable { - typedef T type; -}; - -/// Return type for specialized generic 2-argument half-precision functions. -/// This class template has to be specialized for each valid combination of -/// argument types to provide a corresponding -/// `type` member denoting the appropriate return type. -/// \tparam T first argument type -/// \tparam U first argument type -template -struct result : enable {}; -template <> -struct result { - typedef half type; -}; - -/// \name Classification helpers -/// \{ - -/// Check for infinity. -/// \tparam T argument type (builtin floating point type) -/// \param arg value to query -/// \retval true if infinity -/// \retval false else -template -bool builtin_isinf(T arg) { -#if HALF_ENABLE_CPP11_CMATH - return std::isinf(arg); -#elif defined(_MSC_VER) - return !::_finite(static_cast(arg)) && - !::_isnan(static_cast(arg)); -#else - return arg == std::numeric_limits::infinity() || - arg == -std::numeric_limits::infinity(); -#endif -} - -/// Check for NaN. -/// \tparam T argument type (builtin floating point type) -/// \param arg value to query -/// \retval true if not a number -/// \retval false else -template -bool builtin_isnan(T arg) { -#if HALF_ENABLE_CPP11_CMATH - return std::isnan(arg); -#elif defined(_MSC_VER) - return ::_isnan(static_cast(arg)) != 0; -#else - return arg != arg; -#endif -} - -/// Check sign. -/// \tparam T argument type (builtin floating point type) -/// \param arg value to query -/// \retval true if signbit set -/// \retval false else -template -bool builtin_signbit(T arg) { -#if HALF_ENABLE_CPP11_CMATH - return std::signbit(arg); -#else - return arg < T() || (arg == T() && T(1) / arg < T()); -#endif -} - -/// \} -/// \name Conversion -/// \{ - -/// Convert IEEE single-precision to half-precision. -/// Credit for this goes to [Jeroen van der -/// Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). -/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest -/// rounding -/// \param value single-precision value -/// \return binary representation of half-precision value -template -uint16 float2half_impl(float value, true_type) { - typedef bits::type uint32; - uint32 - bits; // = *reinterpret_cast(&value); //violating - // strict aliasing! - std::memcpy(&bits, &value, sizeof(float)); - /* uint16 hbits = (bits>>16) & 0x8000; + /// Library-defined half-precision literals. + /// Import this namespace to enable half-precision floating point literals: + /// ~~~~{.cpp} + /// using namespace half_float::literal; + /// half_float::half = 4.2_h; + /// ~~~~ + namespace literal + { + half operator"" _h(long double); + } +#endif + + /// \internal + /// \brief Implementation details. + namespace detail + { + #if HALF_ENABLE_CPP11_TYPE_TRAITS + /// Conditional type. + template struct conditional : std::conditional {}; + + /// Helper for tag dispatching. + template struct bool_type : std::integral_constant {}; + using std::true_type; + using std::false_type; + + /// Type traits for floating point types. + template struct is_float : std::is_floating_point {}; + #else + /// Conditional type. + template struct conditional { typedef T type; }; + template struct conditional { typedef F type; }; + + /// Helper for tag dispatching. + template struct bool_type {}; + typedef bool_type true_type; + typedef bool_type false_type; + + /// Type traits for floating point types. + template struct is_float : false_type {}; + template struct is_float : is_float {}; + template struct is_float : is_float {}; + template struct is_float : is_float {}; + template<> struct is_float : true_type {}; + template<> struct is_float : true_type {}; + template<> struct is_float : true_type {}; + #endif + + /// Type traits for floating point bits. + template struct bits { typedef unsigned char type; }; + template struct bits : bits {}; + template struct bits : bits {}; + template struct bits : bits {}; + + #if HALF_ENABLE_CPP11_CSTDINT + /// Unsigned integer of (at least) 16 bits width. + typedef std::uint_least16_t uint16; + + /// Unsigned integer of (at least) 32 bits width. + template<> struct bits { typedef std::uint_least32_t type; }; + + /// Unsigned integer of (at least) 64 bits width. + template<> struct bits { typedef std::uint_least64_t type; }; + #else + /// Unsigned integer of (at least) 16 bits width. + typedef unsigned short uint16; + + /// Unsigned integer of (at least) 32 bits width. + template<> struct bits : conditional::digits>=32,unsigned int,unsigned long> {}; + + #if HALF_ENABLE_CPP11_LONG_LONG + /// Unsigned integer of (at least) 64 bits width. + template<> struct bits : conditional::digits>=64,unsigned long,unsigned long long> {}; + #else + /// Unsigned integer of (at least) 64 bits width. + template<> struct bits { typedef unsigned long type; }; + #endif + #endif + + /// Tag type for binary construction. + struct binary_t {}; + + /// Tag for binary construction. + HALF_CONSTEXPR_CONST binary_t binary = binary_t(); + + /// Temporary half-precision expression. + /// This class represents a half-precision expression which just stores a single-precision value internally. + struct expr + { + /// Conversion constructor. + /// \param f single-precision value to convert + explicit HALF_CONSTEXPR expr(float f) HALF_NOEXCEPT : value_(f) {} + + /// Conversion to single-precision. + /// \return single precision value representing expression value + HALF_CONSTEXPR operator float() const HALF_NOEXCEPT { return value_; } + + private: + /// Internal expression value stored in single-precision. + float value_; + }; + + /// SFINAE helper for generic half-precision functions. + /// This class template has to be specialized for each valid combination of argument types to provide a corresponding + /// `type` member equivalent to \a T. + /// \tparam T type to return + template struct enable {}; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + + /// Return type for specialized generic 2-argument half-precision functions. + /// This class template has to be specialized for each valid combination of argument types to provide a corresponding + /// `type` member denoting the appropriate return type. + /// \tparam T first argument type + /// \tparam U first argument type + template struct result : enable {}; + template<> struct result { typedef half type; }; + + /// \name Classification helpers + /// \{ + + /// Check for infinity. + /// \tparam T argument type (builtin floating point type) + /// \param arg value to query + /// \retval true if infinity + /// \retval false else + template bool builtin_isinf(T arg) + { + #if HALF_ENABLE_CPP11_CMATH + return std::isinf(arg); + #elif defined(_MSC_VER) + return !::_finite(static_cast(arg)) && !::_isnan(static_cast(arg)); + #else + return arg == std::numeric_limits::infinity() || arg == -std::numeric_limits::infinity(); + #endif + } + + /// Check for NaN. + /// \tparam T argument type (builtin floating point type) + /// \param arg value to query + /// \retval true if not a number + /// \retval false else + template bool builtin_isnan(T arg) + { + #if HALF_ENABLE_CPP11_CMATH + return std::isnan(arg); + #elif defined(_MSC_VER) + return ::_isnan(static_cast(arg)) != 0; + #else + return arg != arg; + #endif + } + + /// Check sign. + /// \tparam T argument type (builtin floating point type) + /// \param arg value to query + /// \retval true if signbit set + /// \retval false else + template bool builtin_signbit(T arg) + { + #if HALF_ENABLE_CPP11_CMATH + return std::signbit(arg); + #else + return arg < T() || (arg == T() && T(1)/arg < T()); + #endif + } + + /// \} + /// \name Conversion + /// \{ + + /// Convert IEEE single-precision to half-precision. + /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \param value single-precision value + /// \return binary representation of half-precision value + template uint16 float2half_impl(float value, true_type) + { + typedef bits::type uint32; + uint32 bits;// = *reinterpret_cast(&value); //violating strict aliasing! + std::memcpy(&bits, &value, sizeof(float)); +/* uint16 hbits = (bits>>16) & 0x8000; bits &= 0x7FFFFFFF; int exp = bits >> 23; if(exp == 255) @@ -615,3238 +498,2570 @@ uint16 float2half_impl(float value, true_type) { hbits += ~(hbits>>15) & (s|g); else if(R == std::round_toward_neg_infinity) hbits += (hbits>>15) & (g|s); -*/ static const uint16 - base_table[512] = { - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, - 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100, - 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00, - 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00, - 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, 0x5C00, - 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, - 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100, - 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, - 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00, - 0xC000, 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, - 0xE000, 0xE400, 0xE800, 0xEC00, 0xF000, 0xF400, 0xF800, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00}; - static const unsigned char shift_table[512] = { - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, - 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, - 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 23, - 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, - 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, - 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 13}; - uint16 hbits = - base_table[bits >> 23] + - static_cast((bits & 0x7FFFFF) >> shift_table[bits >> 23]); - if (R == std::round_to_nearest) - hbits += - (((bits & 0x7FFFFF) >> (shift_table[bits >> 23] - 1)) | - (((bits >> 23) & 0xFF) == 102)) & - ((hbits & 0x7C00) != 0x7C00) -#if HALF_ROUND_TIES_TO_EVEN - & (((((static_cast(1) << (shift_table[bits >> 23] - 1)) - 1) & - bits) != 0) | - hbits) -#endif - ; - else if (R == std::round_toward_zero) - hbits -= ((hbits & 0x7FFF) == 0x7C00) & ~shift_table[bits >> 23]; - else if (R == std::round_toward_infinity) - hbits += - ((((bits & 0x7FFFFF & - ((static_cast(1) << (shift_table[bits >> 23])) - 1)) != 0) | - (((bits >> 23) <= 102) & ((bits >> 23) != 0))) & - (hbits < 0x7C00)) - - ((hbits == 0xFC00) & ((bits >> 23) != 511)); - else if (R == std::round_toward_neg_infinity) - hbits += - ((((bits & 0x7FFFFF & - ((static_cast(1) << (shift_table[bits >> 23])) - 1)) != 0) | - (((bits >> 23) <= 358) & ((bits >> 23) != 256))) & - (hbits < 0xFC00) & (hbits >> 15)) - - ((hbits == 0x7C00) & ((bits >> 23) != 255)); - return hbits; -} +*/ static const uint16 base_table[512] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100, + 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00, 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00, + 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100, + 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00, + 0xC000, 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00, 0xF000, 0xF400, 0xF800, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00 }; + static const unsigned char shift_table[512] = { + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13 }; + uint16 hbits = base_table[bits>>23] + static_cast((bits&0x7FFFFF)>>shift_table[bits>>23]); + if(R == std::round_to_nearest) + hbits += (((bits&0x7FFFFF)>>(shift_table[bits>>23]-1))|(((bits>>23)&0xFF)==102)) & ((hbits&0x7C00)!=0x7C00) + #if HALF_ROUND_TIES_TO_EVEN + & (((((static_cast(1)<<(shift_table[bits>>23]-1))-1)&bits)!=0)|hbits) + #endif + ; + else if(R == std::round_toward_zero) + hbits -= ((hbits&0x7FFF)==0x7C00) & ~shift_table[bits>>23]; + else if(R == std::round_toward_infinity) + hbits += ((((bits&0x7FFFFF&((static_cast(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=102)& + ((bits>>23)!=0)))&(hbits<0x7C00)) - ((hbits==0xFC00)&((bits>>23)!=511)); + else if(R == std::round_toward_neg_infinity) + hbits += ((((bits&0x7FFFFF&((static_cast(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=358)& + ((bits>>23)!=256)))&(hbits<0xFC00)&(hbits>>15)) - ((hbits==0x7C00)&((bits>>23)!=255)); + return hbits; + } + + /// Convert IEEE double-precision to half-precision. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \param value double-precision value + /// \return binary representation of half-precision value + template uint16 float2half_impl(double value, true_type) + { + typedef bits::type uint32; + typedef bits::type uint64; + uint64 bits;// = *reinterpret_cast(&value); //violating strict aliasing! + std::memcpy(&bits, &value, sizeof(double)); + uint32 hi = bits >> 32, lo = bits & 0xFFFFFFFF; + uint16 hbits = (hi>>16) & 0x8000; + hi &= 0x7FFFFFFF; + int exp = hi >> 20; + if(exp == 2047) + return hbits | 0x7C00 | (0x3FF&-static_cast((bits&0xFFFFFFFFFFFFF)!=0)); + if(exp > 1038) + { + if(R == std::round_toward_infinity) + return hbits | 0x7C00 - (hbits>>15); + if(R == std::round_toward_neg_infinity) + return hbits | 0x7BFF + (hbits>>15); + return hbits | 0x7BFF + (R!=std::round_toward_zero); + } + int g, s = lo != 0; + if(exp > 1008) + { + g = (hi>>9) & 1; + s |= (hi&0x1FF) != 0; + hbits |= ((exp-1008)<<10) | ((hi>>10)&0x3FF); + } + else if(exp > 997) + { + int i = 1018 - exp; + hi = (hi&0xFFFFF) | 0x100000; + g = (hi>>i) & 1; + s |= (hi&((1L<> (i+1); + } + else + { + g = 0; + s |= hi != 0; + } + if(R == std::round_to_nearest) + #if HALF_ROUND_TIES_TO_EVEN + hbits += g & (s|hbits); + #else + hbits += g; + #endif + else if(R == std::round_toward_infinity) + hbits += ~(hbits>>15) & (s|g); + else if(R == std::round_toward_neg_infinity) + hbits += (hbits>>15) & (g|s); + return hbits; + } + + /// Convert non-IEEE floating point to half-precision. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \tparam T source type (builtin floating point type) + /// \param value floating point value + /// \return binary representation of half-precision value + template uint16 float2half_impl(T value, ...) + { + uint16 hbits = static_cast(builtin_signbit(value)) << 15; + if(value == T()) + return hbits; + if(builtin_isnan(value)) + return hbits | 0x7FFF; + if(builtin_isinf(value)) + return hbits | 0x7C00; + int exp; + std::frexp(value, &exp); + if(exp > 16) + { + if(R == std::round_toward_infinity) + return hbits | (0x7C00 - (hbits>>15)); + else if(R == std::round_toward_neg_infinity) + return hbits | (0x7BFF + (hbits>>15)); + return hbits | (0x7BFF + (R!=std::round_toward_zero)); + } + if(exp < -13) + value = std::ldexp(value, 24); + else + { + value = std::ldexp(value, 11-exp); + hbits |= ((exp+13)<<10); + } + T ival, frac = std::modf(value, &ival); + hbits += static_cast(std::abs(static_cast(ival))); + if(R == std::round_to_nearest) + { + frac = std::abs(frac); + #if HALF_ROUND_TIES_TO_EVEN + hbits += (frac>T(0.5)) | ((frac==T(0.5))&hbits); + #else + hbits += frac >= T(0.5); + #endif + } + else if(R == std::round_toward_infinity) + hbits += frac > T(); + else if(R == std::round_toward_neg_infinity) + hbits += frac < T(); + return hbits; + } + + /// Convert floating point to half-precision. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \tparam T source type (builtin floating point type) + /// \param value floating point value + /// \return binary representation of half-precision value + template uint16 float2half(T value) + { + return float2half_impl(value, bool_type::is_iec559&&sizeof(typename bits::type)==sizeof(T)>()); + } + + /// Convert integer to half-precision floating point. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \tparam S `true` if value negative, `false` else + /// \tparam T type to convert (builtin integer type) + /// \param value non-negative integral value + /// \return binary representation of half-precision value + template uint16 int2half_impl(T value) + { + #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_integral::value, "int to half conversion only supports builtin integer types"); + #endif + if(S) + value = -value; + uint16 bits = S << 15; + if(value > 0xFFFF) + { + if(R == std::round_toward_infinity) + bits |= 0x7C00 - S; + else if(R == std::round_toward_neg_infinity) + bits |= 0x7BFF + S; + else + bits |= 0x7BFF + (R!=std::round_toward_zero); + } + else if(value) + { + unsigned int m = value, exp = 24; + for(; m<0x400; m<<=1,--exp) ; + for(; m>0x7FF; m>>=1,++exp) ; + bits |= (exp<<10) + m; + if(exp > 24) + { + if(R == std::round_to_nearest) + bits += (value>>(exp-25)) & 1 + #if HALF_ROUND_TIES_TO_EVEN + & (((((1<<(exp-25))-1)&value)!=0)|bits) + #endif + ; + else if(R == std::round_toward_infinity) + bits += ((value&((1<<(exp-24))-1))!=0) & !S; + else if(R == std::round_toward_neg_infinity) + bits += ((value&((1<<(exp-24))-1))!=0) & S; + } + } + return bits; + } + + /// Convert integer to half-precision floating point. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \tparam T type to convert (builtin integer type) + /// \param value integral value + /// \return binary representation of half-precision value + template uint16 int2half(T value) + { + return (value<0) ? int2half_impl(value) : int2half_impl(value); + } + + /// Convert half-precision to IEEE single-precision. + /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). + /// \param value binary representation of half-precision value + /// \return single-precision value + inline float half2float_impl(uint16 value, float, true_type) + { + typedef bits::type uint32; +/* uint32 bits = static_cast(value&0x8000) << 16; + int abs = value & 0x7FFF; + if(abs) + { + bits |= 0x38000000 << static_cast(abs>=0x7C00); + for(; abs<0x400; abs<<=1,bits-=0x800000) ; + bits += static_cast(abs) << 13; + } +*/ static const uint32 mantissa_table[2048] = { + 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000, 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000, 0x35700000, + 0x35800000, 0x35880000, 0x35900000, 0x35980000, 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, 0x35F00000, 0x35F80000, + 0x36000000, 0x36040000, 0x36080000, 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000, 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000, + 0x36400000, 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000, 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, 0x36700000, 0x36740000, 0x36780000, 0x367C0000, + 0x36800000, 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000, 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369A0000, 0x369C0000, 0x369E0000, + 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000, + 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000, 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000, + 0x36E00000, 0x36E20000, 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000, 0x36FC0000, 0x36FE0000, + 0x37000000, 0x37010000, 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000, 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000, + 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000, + 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000, 0x372F0000, + 0x37300000, 0x37310000, 0x37320000, 0x37330000, 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, 0x373E0000, 0x373F0000, + 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000, + 0x37500000, 0x37510000, 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000, + 0x37600000, 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000, + 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000, 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000, + 0x37800000, 0x37808000, 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, 0x37870000, 0x37878000, + 0x37880000, 0x37888000, 0x37890000, 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, 0x378F8000, + 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, 0x37960000, 0x37968000, 0x37970000, 0x37978000, + 0x37980000, 0x37988000, 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000, 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000, + 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000, + 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000, 0x37AB0000, 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000, 0x37AF0000, 0x37AF8000, + 0x37B00000, 0x37B08000, 0x37B10000, 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, 0x37B68000, 0x37B70000, 0x37B78000, + 0x37B80000, 0x37B88000, 0x37B90000, 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000, 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000, + 0x37C00000, 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000, + 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, 0x37CF0000, 0x37CF8000, + 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000, 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000, + 0x37D80000, 0x37D88000, 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000, 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000, + 0x37E00000, 0x37E08000, 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000, 0x37E70000, 0x37E78000, + 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000, 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000, + 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000, + 0x37F80000, 0x37F88000, 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, 0x37FF0000, 0x37FF8000, + 0x38000000, 0x38004000, 0x38008000, 0x3800C000, 0x38010000, 0x38014000, 0x38018000, 0x3801C000, 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000, 0x38034000, 0x38038000, 0x3803C000, + 0x38040000, 0x38044000, 0x38048000, 0x3804C000, 0x38050000, 0x38054000, 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, 0x38070000, 0x38074000, 0x38078000, 0x3807C000, + 0x38080000, 0x38084000, 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000, 0x380A0000, 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000, + 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000, 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, 0x380FC000, + 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, 0x38114000, 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000, 0x38130000, 0x38134000, 0x38138000, 0x3813C000, + 0x38140000, 0x38144000, 0x38148000, 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, 0x38174000, 0x38178000, 0x3817C000, + 0x38180000, 0x38184000, 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000, 0x3819C000, 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000, + 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000, 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000, + 0x38200000, 0x38204000, 0x38208000, 0x3820C000, 0x38210000, 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000, 0x38228000, 0x3822C000, 0x38230000, 0x38234000, 0x38238000, 0x3823C000, + 0x38240000, 0x38244000, 0x38248000, 0x3824C000, 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000, + 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000, 0x38298000, 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000, 0x382B4000, 0x382B8000, 0x382BC000, + 0x382C0000, 0x382C4000, 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000, + 0x38300000, 0x38304000, 0x38308000, 0x3830C000, 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000, 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000, + 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000, 0x38358000, 0x3835C000, 0x38360000, 0x38364000, 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000, 0x3837C000, + 0x38380000, 0x38384000, 0x38388000, 0x3838C000, 0x38390000, 0x38394000, 0x38398000, 0x3839C000, 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, 0x383B8000, 0x383BC000, + 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000, 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000, 0x383E8000, 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000, + 0x38400000, 0x38404000, 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000, 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, 0x38430000, 0x38434000, 0x38438000, 0x3843C000, + 0x38440000, 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, 0x3845C000, 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000, 0x38478000, 0x3847C000, + 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000, 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000, + 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000, + 0x38500000, 0x38504000, 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000, 0x38538000, 0x3853C000, + 0x38540000, 0x38544000, 0x38548000, 0x3854C000, 0x38550000, 0x38554000, 0x38558000, 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000, 0x38570000, 0x38574000, 0x38578000, 0x3857C000, + 0x38580000, 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000, + 0x385C0000, 0x385C4000, 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000, 0x385FC000, + 0x38600000, 0x38604000, 0x38608000, 0x3860C000, 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000, 0x38638000, 0x3863C000, + 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, 0x38654000, 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000, 0x38670000, 0x38674000, 0x38678000, 0x3867C000, + 0x38680000, 0x38684000, 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000, 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000, + 0x386C0000, 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000, + 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000, 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000, 0x38730000, 0x38734000, 0x38738000, 0x3873C000, + 0x38740000, 0x38744000, 0x38748000, 0x3874C000, 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000, 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, 0x38778000, 0x3877C000, + 0x38780000, 0x38784000, 0x38788000, 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000, + 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000, 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000, + 0x38000000, 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800A000, 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000, + 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000, + 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, 0x38050000, 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000, 0x3805C000, 0x3805E000, + 0x38060000, 0x38062000, 0x38064000, 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807A000, 0x3807C000, 0x3807E000, + 0x38080000, 0x38082000, 0x38084000, 0x38086000, 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000, 0x38094000, 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000, + 0x380A0000, 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000, + 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, 0x380CE000, 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, 0x380DC000, 0x380DE000, + 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000, 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000, + 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000, + 0x38120000, 0x38122000, 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000, 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000, 0x3813C000, 0x3813E000, + 0x38140000, 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000, 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000, 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000, + 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, 0x38176000, 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000, + 0x38180000, 0x38182000, 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, 0x3819C000, 0x3819E000, + 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, 0x381BC000, 0x381BE000, + 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000, + 0x381E0000, 0x381E2000, 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000, + 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, 0x3821E000, + 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000, 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823A000, 0x3823C000, 0x3823E000, + 0x38240000, 0x38242000, 0x38244000, 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825A000, 0x3825C000, 0x3825E000, + 0x38260000, 0x38262000, 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, 0x38272000, 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000, + 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000, + 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000, 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, 0x382BC000, 0x382BE000, + 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000, 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000, + 0x382E0000, 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000, 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000, 0x382FA000, 0x382FC000, 0x382FE000, + 0x38300000, 0x38302000, 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831A000, 0x3831C000, 0x3831E000, + 0x38320000, 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000, 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000, + 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000, + 0x38360000, 0x38362000, 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, 0x3837C000, 0x3837E000, + 0x38380000, 0x38382000, 0x38384000, 0x38386000, 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000, + 0x383A0000, 0x383A2000, 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000, + 0x383C0000, 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000, 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, 0x383DE000, + 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000, 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, 0x383FE000, + 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000, + 0x38420000, 0x38422000, 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000, 0x3843C000, 0x3843E000, + 0x38440000, 0x38442000, 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000, 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000, + 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000, + 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000, 0x3849E000, + 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, 0x384BE000, + 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000, 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000, 0x384DA000, 0x384DC000, 0x384DE000, + 0x384E0000, 0x384E2000, 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000, + 0x38500000, 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000, + 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000, 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, 0x38538000, 0x3853A000, 0x3853C000, 0x3853E000, + 0x38540000, 0x38542000, 0x38544000, 0x38546000, 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, 0x3855C000, 0x3855E000, + 0x38560000, 0x38562000, 0x38564000, 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000, + 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000, 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000, + 0x385A0000, 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, 0x385BA000, 0x385BC000, 0x385BE000, + 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000, + 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000, 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000, + 0x38600000, 0x38602000, 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000, 0x3861C000, 0x3861E000, + 0x38620000, 0x38622000, 0x38624000, 0x38626000, 0x38628000, 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000, + 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000, + 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000, 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000, 0x3867E000, + 0x38680000, 0x38682000, 0x38684000, 0x38686000, 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869A000, 0x3869C000, 0x3869E000, + 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000, + 0x386C0000, 0x386C2000, 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000, 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000, + 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000, 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000, + 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000, + 0x38720000, 0x38722000, 0x38724000, 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, 0x3873C000, 0x3873E000, + 0x38740000, 0x38742000, 0x38744000, 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, 0x3875E000, + 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000, 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000, + 0x38780000, 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000, + 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000, + 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000, 0x387CC000, 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000, 0x387DC000, 0x387DE000, + 0x387E0000, 0x387E2000, 0x387E4000, 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, 0x387FA000, 0x387FC000, 0x387FE000 }; + static const uint32 exponent_table[64] = { + 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, 0x06000000, 0x06800000, 0x07000000, 0x07800000, + 0x08000000, 0x08800000, 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, 0x0F000000, 0x47800000, + 0x80000000, 0x80800000, 0x81000000, 0x81800000, 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000, + 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000 }; + static const unsigned short offset_table[64] = { + 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, + 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024 }; + uint32 bits = mantissa_table[offset_table[value>>10]+(value&0x3FF)] + exponent_table[value>>10]; +// return *reinterpret_cast(&bits); //violating strict aliasing! + float out; + std::memcpy(&out, &bits, sizeof(float)); + return out; + } + + /// Convert half-precision to IEEE double-precision. + /// \param value binary representation of half-precision value + /// \return double-precision value + inline double half2float_impl(uint16 value, double, true_type) + { + typedef bits::type uint32; + typedef bits::type uint64; + uint32 hi = static_cast(value&0x8000) << 16; + int abs = value & 0x7FFF; + if(abs) + { + hi |= 0x3F000000 << static_cast(abs>=0x7C00); + for(; abs<0x400; abs<<=1,hi-=0x100000) ; + hi += static_cast(abs) << 10; + } + uint64 bits = static_cast(hi) << 32; +// return *reinterpret_cast(&bits); //violating strict aliasing! + double out; + std::memcpy(&out, &bits, sizeof(double)); + return out; + } + + /// Convert half-precision to non-IEEE floating point. + /// \tparam T type to convert to (builtin integer type) + /// \param value binary representation of half-precision value + /// \return floating point value + template T half2float_impl(uint16 value, T, ...) + { + T out; + int abs = value & 0x7FFF; + if(abs > 0x7C00) + out = std::numeric_limits::has_quiet_NaN ? std::numeric_limits::quiet_NaN() : T(); + else if(abs == 0x7C00) + out = std::numeric_limits::has_infinity ? std::numeric_limits::infinity() : std::numeric_limits::max(); + else if(abs > 0x3FF) + out = std::ldexp(static_cast((abs&0x3FF)|0x400), (abs>>10)-25); + else + out = std::ldexp(static_cast(abs), -24); + return (value&0x8000) ? -out : out; + } + + /// Convert half-precision to floating point. + /// \tparam T type to convert to (builtin integer type) + /// \param value binary representation of half-precision value + /// \return floating point value + template T half2float(uint16 value) + { + return half2float_impl(value, T(), bool_type::is_iec559&&sizeof(typename bits::type)==sizeof(T)>()); + } + + /// Convert half-precision floating point to integer. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \tparam E `true` for round to even, `false` for round away from zero + /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) + /// \param value binary representation of half-precision value + /// \return integral value + template T half2int_impl(uint16 value) + { + #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_integral::value, "half to int conversion only supports builtin integer types"); + #endif + unsigned int e = value & 0x7FFF; + if(e >= 0x7C00) + return (value&0x8000) ? std::numeric_limits::min() : std::numeric_limits::max(); + if(e < 0x3800) + { + if(R == std::round_toward_infinity) + return T(~(value>>15)&(e!=0)); + else if(R == std::round_toward_neg_infinity) + return -T(value>0x8000); + return T(); + } + unsigned int m = (value&0x3FF) | 0x400; + e >>= 10; + if(e < 25) + { + if(R == std::round_to_nearest) + m += (1<<(24-e)) - (~(m>>(25-e))&E); + else if(R == std::round_toward_infinity) + m += ((value>>15)-1) & ((1<<(25-e))-1U); + else if(R == std::round_toward_neg_infinity) + m += -(value>>15) & ((1<<(25-e))-1U); + m >>= 25 - e; + } + else + m <<= e - 25; + return (value&0x8000) ? -static_cast(m) : static_cast(m); + } + + /// Convert half-precision floating point to integer. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) + /// \param value binary representation of half-precision value + /// \return integral value + template T half2int(uint16 value) { return half2int_impl(value); } + + /// Convert half-precision floating point to integer using round-to-nearest-away-from-zero. + /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) + /// \param value binary representation of half-precision value + /// \return integral value + template T half2int_up(uint16 value) { return half2int_impl(value); } + + /// Round half-precision number to nearest integer value. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \tparam E `true` for round to even, `false` for round away from zero + /// \param value binary representation of half-precision value + /// \return half-precision bits for nearest integral value + template uint16 round_half_impl(uint16 value) + { + unsigned int e = value & 0x7FFF; + uint16 result = value; + if(e < 0x3C00) + { + result &= 0x8000; + if(R == std::round_to_nearest) + result |= 0x3C00U & -(e>=(0x3800+E)); + else if(R == std::round_toward_infinity) + result |= 0x3C00U & -(~(value>>15)&(e!=0)); + else if(R == std::round_toward_neg_infinity) + result |= 0x3C00U & -(value>0x8000); + } + else if(e < 0x6400) + { + e = 25 - (e>>10); + unsigned int mask = (1<>e)&E); + else if(R == std::round_toward_infinity) + result += mask & ((value>>15)-1); + else if(R == std::round_toward_neg_infinity) + result += mask & -(value>>15); + result &= ~mask; + } + return result; + } + + /// Round half-precision number to nearest integer value. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \param value binary representation of half-precision value + /// \return half-precision bits for nearest integral value + template uint16 round_half(uint16 value) { return round_half_impl(value); } + + /// Round half-precision number to nearest integer value using round-to-nearest-away-from-zero. + /// \param value binary representation of half-precision value + /// \return half-precision bits for nearest integral value + inline uint16 round_half_up(uint16 value) { return round_half_impl(value); } + /// \} + + struct functions; + template struct unary_specialized; + template struct binary_specialized; + template struct half_caster; + } + + /// Half-precision floating point type. + /// This class implements an IEEE-conformant half-precision floating point type with the usual arithmetic operators and + /// conversions. It is implicitly convertible to single-precision floating point, which makes artihmetic expressions and + /// functions with mixed-type operands to be of the most precise operand type. Additionally all arithmetic operations + /// (and many mathematical functions) are carried out in single-precision internally. All conversions from single- to + /// half-precision are done using the library's default rounding mode, but temporary results inside chained arithmetic + /// expressions are kept in single-precision as long as possible (while of course still maintaining a strong half-precision type). + /// + /// According to the C++98/03 definition, the half type is not a POD type. But according to C++11's less strict and + /// extended definitions it is both a standard layout type and a trivially copyable type (even if not a POD type), which + /// means it can be standard-conformantly copied using raw binary copies. But in this context some more words about the + /// actual size of the type. Although the half is representing an IEEE 16-bit type, it does not neccessarily have to be of + /// exactly 16-bits size. But on any reasonable implementation the actual binary representation of this type will most + /// probably not ivolve any additional "magic" or padding beyond the simple binary representation of the underlying 16-bit + /// IEEE number, even if not strictly guaranteed by the standard. But even then it only has an actual size of 16 bits if + /// your C++ implementation supports an unsigned integer type of exactly 16 bits width. But this should be the case on + /// nearly any reasonable platform. + /// + /// So if your C++ implementation is not totally exotic or imposes special alignment requirements, it is a reasonable + /// assumption that the data of a half is just comprised of the 2 bytes of the underlying IEEE representation. + class half + { + friend struct detail::functions; + friend struct detail::unary_specialized; + friend struct detail::binary_specialized; + template friend struct detail::half_caster; + friend class std::numeric_limits; + #if HALF_ENABLE_CPP11_HASH + friend struct std::hash; + #endif + #if HALF_ENABLE_CPP11_USER_LITERALS + friend half literal::operator"" _h(long double); + #endif + + public: + /// Default constructor. + /// This initializes the half to 0. Although this does not match the builtin types' default-initialization semantics + /// and may be less efficient than no initialization, it is needed to provide proper value-initialization semantics. + HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {} + + /// Copy constructor. + /// \tparam T type of concrete half expression + /// \param rhs half expression to copy from + half(detail::expr rhs) : data_(detail::float2half(static_cast(rhs))) {} + + /// Conversion constructor. + /// \param rhs float to convert + explicit half(float rhs) : data_(detail::float2half(rhs)) {} + + /// Conversion to single-precision. + /// \return single precision value representing expression value + operator float() const { return detail::half2float(data_); } + + /// Assignment operator. + /// \tparam T type of concrete half expression + /// \param rhs half expression to copy from + /// \return reference to this half + half& operator=(detail::expr rhs) { return *this = static_cast(rhs); } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to add + /// \return reference to this half + template typename detail::enable::type operator+=(T rhs) { return *this += static_cast(rhs); } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to subtract + /// \return reference to this half + template typename detail::enable::type operator-=(T rhs) { return *this -= static_cast(rhs); } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to multiply with + /// \return reference to this half + template typename detail::enable::type operator*=(T rhs) { return *this *= static_cast(rhs); } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to divide by + /// \return reference to this half + template typename detail::enable::type operator/=(T rhs) { return *this /= static_cast(rhs); } + + /// Assignment operator. + /// \param rhs single-precision value to copy from + /// \return reference to this half + half& operator=(float rhs) { data_ = detail::float2half(rhs); return *this; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to add + /// \return reference to this half + half& operator+=(float rhs) { data_ = detail::float2half(detail::half2float(data_)+rhs); return *this; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to subtract + /// \return reference to this half + half& operator-=(float rhs) { data_ = detail::float2half(detail::half2float(data_)-rhs); return *this; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to multiply with + /// \return reference to this half + half& operator*=(float rhs) { data_ = detail::float2half(detail::half2float(data_)*rhs); return *this; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to divide by + /// \return reference to this half + half& operator/=(float rhs) { data_ = detail::float2half(detail::half2float(data_)/rhs); return *this; } + + /// Prefix increment. + /// \return incremented half value + half& operator++() { return *this += 1.0f; } + + /// Prefix decrement. + /// \return decremented half value + half& operator--() { return *this -= 1.0f; } + + /// Postfix increment. + /// \return non-incremented half value + half operator++(int) { half out(*this); ++*this; return out; } + + /// Postfix decrement. + /// \return non-decremented half value + half operator--(int) { half out(*this); --*this; return out; } + + private: + /// Rounding mode to use + static const std::float_round_style round_style = (std::float_round_style)(HALF_ROUND_STYLE); + + /// Constructor. + /// \param bits binary representation to set half to + HALF_CONSTEXPR half(detail::binary_t, detail::uint16 bits) HALF_NOEXCEPT : data_(bits) {} + + /// Internal binary representation + detail::uint16 data_; + }; -/// Convert IEEE double-precision to half-precision. -/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest -/// rounding -/// \param value double-precision value -/// \return binary representation of half-precision value -template -uint16 float2half_impl(double value, true_type) { - typedef bits::type uint32; - typedef bits::type uint64; - uint64 - bits; // = *reinterpret_cast(&value); //violating - // strict aliasing! - std::memcpy(&bits, &value, sizeof(double)); - uint32 hi = bits >> 32, lo = bits & 0xFFFFFFFF; - uint16 hbits = (hi >> 16) & 0x8000; - hi &= 0x7FFFFFFF; - int exp = hi >> 20; - if (exp == 2047) - return hbits | 0x7C00 | - (0x3FF & -static_cast((bits & 0xFFFFFFFFFFFFF) != 0)); - if (exp > 1038) { - if (R == std::round_toward_infinity) return hbits | 0x7C00 - (hbits >> 15); - if (R == std::round_toward_neg_infinity) - return hbits | 0x7BFF + (hbits >> 15); - return hbits | 0x7BFF + (R != std::round_toward_zero); - } - int g, s = lo != 0; - if (exp > 1008) { - g = (hi >> 9) & 1; - s |= (hi & 0x1FF) != 0; - hbits |= ((exp - 1008) << 10) | ((hi >> 10) & 0x3FF); - } else if (exp > 997) { - int i = 1018 - exp; - hi = (hi & 0xFFFFF) | 0x100000; - g = (hi >> i) & 1; - s |= (hi & ((1L << i) - 1)) != 0; - hbits |= hi >> (i + 1); - } else { - g = 0; - s |= hi != 0; - } - if (R == std::round_to_nearest) -#if HALF_ROUND_TIES_TO_EVEN - hbits += g & (s | hbits); -#else - hbits += g; -#endif - else if (R == std::round_toward_infinity) - hbits += ~(hbits >> 15) & (s | g); - else if (R == std::round_toward_neg_infinity) - hbits += (hbits >> 15) & (g | s); - return hbits; -} +#if HALF_ENABLE_CPP11_USER_LITERALS + namespace literal + { + /// Half literal. + /// While this returns an actual half-precision value, half literals can unfortunately not be constant expressions due + /// to rather involved conversions. + /// \param value literal value + /// \return half with given value (if representable) + inline half operator"" _h(long double value) { return half(detail::binary, detail::float2half(value)); } + } +#endif + + namespace detail + { + /// Wrapper implementing unspecialized half-precision functions. + struct functions + { + /// Addition implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision sum stored in single-precision + static expr plus(float x, float y) { return expr(x+y); } + + /// Subtraction implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision difference stored in single-precision + static expr minus(float x, float y) { return expr(x-y); } + + /// Multiplication implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision product stored in single-precision + static expr multiplies(float x, float y) { return expr(x*y); } + + /// Division implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision quotient stored in single-precision + static expr divides(float x, float y) { return expr(x/y); } + + /// Output implementation. + /// \param out stream to write to + /// \param arg value to write + /// \return reference to stream + template static std::basic_ostream& write(std::basic_ostream &out, float arg) { return out << arg; } + + /// Input implementation. + /// \param in stream to read from + /// \param arg half to read into + /// \return reference to stream + template static std::basic_istream& read(std::basic_istream &in, half &arg) + { + float f; + if(in >> f) + arg = f; + return in; + } -/// Convert non-IEEE floating point to half-precision. -/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest -/// rounding -/// \tparam T source type (builtin floating point type) -/// \param value floating point value -/// \return binary representation of half-precision value -template -uint16 float2half_impl(T value, ...) { - uint16 hbits = static_cast(builtin_signbit(value)) << 15; - if (value == T()) return hbits; - if (builtin_isnan(value)) return hbits | 0x7FFF; - if (builtin_isinf(value)) return hbits | 0x7C00; - int exp; - std::frexp(value, &exp); - if (exp > 16) { - if (R == std::round_toward_infinity) - return hbits | (0x7C00 - (hbits >> 15)); - else if (R == std::round_toward_neg_infinity) - return hbits | (0x7BFF + (hbits >> 15)); - return hbits | (0x7BFF + (R != std::round_toward_zero)); - } - if (exp < -13) - value = std::ldexp(value, 24); - else { - value = std::ldexp(value, 11 - exp); - hbits |= ((exp + 13) << 10); - } - T ival, frac = std::modf(value, &ival); - hbits += static_cast(std::abs(static_cast(ival))); - if (R == std::round_to_nearest) { - frac = std::abs(frac); -#if HALF_ROUND_TIES_TO_EVEN - hbits += (frac > T(0.5)) | ((frac == T(0.5)) & hbits); -#else - hbits += frac >= T(0.5); -#endif - } else if (R == std::round_toward_infinity) - hbits += frac > T(); - else if (R == std::round_toward_neg_infinity) - hbits += frac < T(); - return hbits; -} + /// Modulo implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision division remainder stored in single-precision + static expr fmod(float x, float y) { return expr(std::fmod(x, y)); } + + /// Remainder implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision division remainder stored in single-precision + static expr remainder(float x, float y) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::remainder(x, y)); + #else + if(builtin_isnan(x) || builtin_isnan(y)) + return expr(std::numeric_limits::quiet_NaN()); + float ax = std::fabs(x), ay = std::fabs(y); + if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) + return expr(std::numeric_limits::quiet_NaN()); + if(ay >= 65536.0f) + return expr(x); + if(ax == ay) + return expr(builtin_signbit(x) ? -0.0f : 0.0f); + ax = std::fmod(ax, ay+ay); + float y2 = 0.5f * ay; + if(ax > y2) + { + ax -= ay; + if(ax >= y2) + ax -= ay; + } + return expr(builtin_signbit(x) ? -ax : ax); + #endif + } -/// Convert floating point to half-precision. -/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest -/// rounding -/// \tparam T source type (builtin floating point type) -/// \param value floating point value -/// \return binary representation of half-precision value -template -uint16 float2half(T value) { - return float2half_impl( - value, bool_type < std::numeric_limits::is_iec559 && - sizeof(typename bits::type) == sizeof(T) > ()); -} + /// Remainder implementation. + /// \param x first operand + /// \param y second operand + /// \param quo address to store quotient bits at + /// \return Half-precision division remainder stored in single-precision + static expr remquo(float x, float y, int *quo) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::remquo(x, y, quo)); + #else + if(builtin_isnan(x) || builtin_isnan(y)) + return expr(std::numeric_limits::quiet_NaN()); + bool sign = builtin_signbit(x), qsign = static_cast(sign^builtin_signbit(y)); + float ax = std::fabs(x), ay = std::fabs(y); + if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) + return expr(std::numeric_limits::quiet_NaN()); + if(ay >= 65536.0f) + return expr(x); + if(ax == ay) + return *quo = qsign ? -1 : 1, expr(sign ? -0.0f : 0.0f); + ax = std::fmod(ax, 8.0f*ay); + int cquo = 0; + if(ax >= 4.0f * ay) + { + ax -= 4.0f * ay; + cquo += 4; + } + if(ax >= 2.0f * ay) + { + ax -= 2.0f * ay; + cquo += 2; + } + float y2 = 0.5f * ay; + if(ax > y2) + { + ax -= ay; + ++cquo; + if(ax >= y2) + { + ax -= ay; + ++cquo; + } + } + return *quo = qsign ? -cquo : cquo, expr(sign ? -ax : ax); + #endif + } -/// Convert integer to half-precision floating point. -/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest -/// rounding -/// \tparam S `true` if value negative, `false` else -/// \tparam T type to convert (builtin integer type) -/// \param value non-negative integral value -/// \return binary representation of half-precision value -template -uint16 int2half_impl(T value) { -#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_integral::value, - "int to half conversion only supports builtin integer types"); -#endif - if (S) value = -value; - uint16 bits = S << 15; - if (value > 0xFFFF) { - if (R == std::round_toward_infinity) - bits |= 0x7C00 - S; - else if (R == std::round_toward_neg_infinity) - bits |= 0x7BFF + S; - else - bits |= 0x7BFF + (R != std::round_toward_zero); - } else if (value) { - unsigned int m = value, exp = 24; - for (; m < 0x400; m <<= 1, --exp) - ; - for (; m > 0x7FF; m >>= 1, ++exp) - ; - bits |= (exp << 10) + m; - if (exp > 24) { - if (R == std::round_to_nearest) - bits += (value >> (exp - 25)) & 1 -#if HALF_ROUND_TIES_TO_EVEN - & (((((1 << (exp - 25)) - 1) & value) != 0) | bits) -#endif - ; - else if (R == std::round_toward_infinity) - bits += ((value & ((1 << (exp - 24)) - 1)) != 0) & !S; - else if (R == std::round_toward_neg_infinity) - bits += ((value & ((1 << (exp - 24)) - 1)) != 0) & S; - } - } - return bits; -} + /// Positive difference implementation. + /// \param x first operand + /// \param y second operand + /// \return Positive difference stored in single-precision + static expr fdim(float x, float y) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::fdim(x, y)); + #else + return expr((x<=y) ? 0.0f : (x-y)); + #endif + } -/// Convert integer to half-precision floating point. -/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest -/// rounding -/// \tparam T type to convert (builtin integer type) -/// \param value integral value -/// \return binary representation of half-precision value -template -uint16 int2half(T value) { - return (value < 0) ? int2half_impl(value) - : int2half_impl(value); -} + /// Fused multiply-add implementation. + /// \param x first operand + /// \param y second operand + /// \param z third operand + /// \return \a x * \a y + \a z stored in single-precision + static expr fma(float x, float y, float z) + { + #if HALF_ENABLE_CPP11_CMATH && defined(FP_FAST_FMAF) + return expr(std::fma(x, y, z)); + #else + return expr(x*y+z); + #endif + } -/// Convert half-precision to IEEE single-precision. -/// Credit for this goes to [Jeroen van der -/// Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). -/// \param value binary representation of half-precision value -/// \return single-precision value -inline float half2float_impl(uint16 value, float, true_type) { - typedef bits::type uint32; - /* uint32 bits = static_cast(value&0x8000) << 16; - int abs = value & 0x7FFF; - if(abs) + /// Get NaN. + /// \return Half-precision quiet NaN + static half nanh() { return half(binary, 0x7FFF); } + + /// Exponential implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr exp(float arg) { return expr(std::exp(arg)); } + + /// Exponential implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr expm1(float arg) { - bits |= 0x38000000 << static_cast(abs>=0x7C00); - for(; abs<0x400; abs<<=1,bits-=0x800000) ; - bits += static_cast(abs) << 13; + #if HALF_ENABLE_CPP11_CMATH + return expr(std::expm1(arg)); + #else + return expr(static_cast(std::exp(static_cast(arg))-1.0)); + #endif } -*/ static const uint32 - mantissa_table[2048] = { - 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, - 0x34A00000, 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, - 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000, - 0x35700000, 0x35800000, 0x35880000, 0x35900000, 0x35980000, - 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000, - 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, - 0x35F00000, 0x35F80000, 0x36000000, 0x36040000, 0x36080000, - 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000, - 0x36200000, 0x36240000, 0x36280000, 0x362C0000, 0x36300000, - 0x36340000, 0x36380000, 0x363C0000, 0x36400000, 0x36440000, - 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000, - 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, - 0x36700000, 0x36740000, 0x36780000, 0x367C0000, 0x36800000, - 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000, - 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000, - 0x36960000, 0x36980000, 0x369A0000, 0x369C0000, 0x369E0000, - 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, 0x36A80000, - 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, - 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, - 0x36BE0000, 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, - 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000, - 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000, - 0x36DC0000, 0x36DE0000, 0x36E00000, 0x36E20000, 0x36E40000, - 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, - 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, - 0x36FA0000, 0x36FC0000, 0x36FE0000, 0x37000000, 0x37010000, - 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000, - 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000, - 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000, 0x37100000, - 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, - 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, - 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000, - 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000, - 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000, - 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000, - 0x372F0000, 0x37300000, 0x37310000, 0x37320000, 0x37330000, - 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, - 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, - 0x373E0000, 0x373F0000, 0x37400000, 0x37410000, 0x37420000, - 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000, - 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, 0x374C0000, - 0x374D0000, 0x374E0000, 0x374F0000, 0x37500000, 0x37510000, - 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, - 0x37570000, 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, - 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000, 0x37600000, - 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, - 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376A0000, - 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000, - 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, - 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, - 0x377A0000, 0x377B0000, 0x377C0000, 0x377D0000, 0x377E0000, - 0x377F0000, 0x37800000, 0x37808000, 0x37810000, 0x37818000, - 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, - 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, - 0x37870000, 0x37878000, 0x37880000, 0x37888000, 0x37890000, - 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, - 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000, - 0x378E8000, 0x378F0000, 0x378F8000, 0x37900000, 0x37908000, - 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000, - 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, - 0x37960000, 0x37968000, 0x37970000, 0x37978000, 0x37980000, - 0x37988000, 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, - 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000, - 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000, - 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, 0x37A20000, - 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, - 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, - 0x37A78000, 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, - 0x37AA0000, 0x37AA8000, 0x37AB0000, 0x37AB8000, 0x37AC0000, - 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000, - 0x37AF0000, 0x37AF8000, 0x37B00000, 0x37B08000, 0x37B10000, - 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, - 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, - 0x37B68000, 0x37B70000, 0x37B78000, 0x37B80000, 0x37B88000, - 0x37B90000, 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000, - 0x37BB8000, 0x37BC0000, 0x37BC8000, 0x37BD0000, 0x37BD8000, - 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000, 0x37C00000, - 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, - 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, - 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000, - 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000, - 0x37CA8000, 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000, - 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, 0x37CF0000, - 0x37CF8000, 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, - 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, - 0x37D48000, 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, - 0x37D70000, 0x37D78000, 0x37D80000, 0x37D88000, 0x37D90000, - 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000, - 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000, - 0x37DE8000, 0x37DF0000, 0x37DF8000, 0x37E00000, 0x37E08000, - 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, - 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, - 0x37E60000, 0x37E68000, 0x37E70000, 0x37E78000, 0x37E80000, - 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, 0x37EA8000, - 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000, - 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000, - 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, - 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, - 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000, - 0x37F78000, 0x37F80000, 0x37F88000, 0x37F90000, 0x37F98000, - 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, 0x37FC0000, - 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, - 0x37FF0000, 0x37FF8000, 0x38000000, 0x38004000, 0x38008000, - 0x3800C000, 0x38010000, 0x38014000, 0x38018000, 0x3801C000, - 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000, - 0x38034000, 0x38038000, 0x3803C000, 0x38040000, 0x38044000, - 0x38048000, 0x3804C000, 0x38050000, 0x38054000, 0x38058000, - 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, - 0x38070000, 0x38074000, 0x38078000, 0x3807C000, 0x38080000, - 0x38084000, 0x38088000, 0x3808C000, 0x38090000, 0x38094000, - 0x38098000, 0x3809C000, 0x380A0000, 0x380A4000, 0x380A8000, - 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000, - 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000, - 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, - 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, - 0x380FC000, 0x38100000, 0x38104000, 0x38108000, 0x3810C000, - 0x38110000, 0x38114000, 0x38118000, 0x3811C000, 0x38120000, - 0x38124000, 0x38128000, 0x3812C000, 0x38130000, 0x38134000, - 0x38138000, 0x3813C000, 0x38140000, 0x38144000, 0x38148000, - 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, - 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, - 0x38174000, 0x38178000, 0x3817C000, 0x38180000, 0x38184000, - 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000, - 0x3819C000, 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000, - 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000, 0x381C0000, - 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, - 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, - 0x381EC000, 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000, - 0x38200000, 0x38204000, 0x38208000, 0x3820C000, 0x38210000, - 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000, - 0x38228000, 0x3822C000, 0x38230000, 0x38234000, 0x38238000, - 0x3823C000, 0x38240000, 0x38244000, 0x38248000, 0x3824C000, - 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, - 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000, - 0x38278000, 0x3827C000, 0x38280000, 0x38284000, 0x38288000, - 0x3828C000, 0x38290000, 0x38294000, 0x38298000, 0x3829C000, - 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000, - 0x382B4000, 0x382B8000, 0x382BC000, 0x382C0000, 0x382C4000, - 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, - 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, - 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000, 0x38300000, - 0x38304000, 0x38308000, 0x3830C000, 0x38310000, 0x38314000, - 0x38318000, 0x3831C000, 0x38320000, 0x38324000, 0x38328000, - 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000, - 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, - 0x38354000, 0x38358000, 0x3835C000, 0x38360000, 0x38364000, - 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000, - 0x3837C000, 0x38380000, 0x38384000, 0x38388000, 0x3838C000, - 0x38390000, 0x38394000, 0x38398000, 0x3839C000, 0x383A0000, - 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, - 0x383B8000, 0x383BC000, 0x383C0000, 0x383C4000, 0x383C8000, - 0x383CC000, 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, - 0x383E0000, 0x383E4000, 0x383E8000, 0x383EC000, 0x383F0000, - 0x383F4000, 0x383F8000, 0x383FC000, 0x38400000, 0x38404000, - 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000, - 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, - 0x38430000, 0x38434000, 0x38438000, 0x3843C000, 0x38440000, - 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000, - 0x38458000, 0x3845C000, 0x38460000, 0x38464000, 0x38468000, - 0x3846C000, 0x38470000, 0x38474000, 0x38478000, 0x3847C000, - 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000, - 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, - 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, - 0x384BC000, 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, - 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000, - 0x384E4000, 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000, - 0x384F8000, 0x384FC000, 0x38500000, 0x38504000, 0x38508000, - 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, - 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, - 0x38534000, 0x38538000, 0x3853C000, 0x38540000, 0x38544000, - 0x38548000, 0x3854C000, 0x38550000, 0x38554000, 0x38558000, - 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000, - 0x38570000, 0x38574000, 0x38578000, 0x3857C000, 0x38580000, - 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, - 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, - 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000, - 0x385C0000, 0x385C4000, 0x385C8000, 0x385CC000, 0x385D0000, - 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, 0x385E4000, - 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000, - 0x385FC000, 0x38600000, 0x38604000, 0x38608000, 0x3860C000, - 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, - 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000, - 0x38638000, 0x3863C000, 0x38640000, 0x38644000, 0x38648000, - 0x3864C000, 0x38650000, 0x38654000, 0x38658000, 0x3865C000, - 0x38660000, 0x38664000, 0x38668000, 0x3866C000, 0x38670000, - 0x38674000, 0x38678000, 0x3867C000, 0x38680000, 0x38684000, - 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, - 0x3869C000, 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, - 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000, 0x386C0000, - 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000, - 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, 0x386E8000, - 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000, - 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, - 0x38714000, 0x38718000, 0x3871C000, 0x38720000, 0x38724000, - 0x38728000, 0x3872C000, 0x38730000, 0x38734000, 0x38738000, - 0x3873C000, 0x38740000, 0x38744000, 0x38748000, 0x3874C000, - 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000, - 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, - 0x38778000, 0x3877C000, 0x38780000, 0x38784000, 0x38788000, - 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000, - 0x387A0000, 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000, - 0x387B4000, 0x387B8000, 0x387BC000, 0x387C0000, 0x387C4000, - 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000, - 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, - 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000, 0x38000000, - 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800A000, - 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000, - 0x38016000, 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000, - 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000, - 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, - 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, - 0x3803E000, 0x38040000, 0x38042000, 0x38044000, 0x38046000, - 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, 0x38050000, - 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000, - 0x3805C000, 0x3805E000, 0x38060000, 0x38062000, 0x38064000, - 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, - 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, - 0x3807A000, 0x3807C000, 0x3807E000, 0x38080000, 0x38082000, - 0x38084000, 0x38086000, 0x38088000, 0x3808A000, 0x3808C000, - 0x3808E000, 0x38090000, 0x38092000, 0x38094000, 0x38096000, - 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000, 0x380A0000, - 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, - 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, - 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000, - 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000, - 0x380CA000, 0x380CC000, 0x380CE000, 0x380D0000, 0x380D2000, - 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, 0x380DC000, - 0x380DE000, 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, - 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, - 0x380F2000, 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, - 0x380FC000, 0x380FE000, 0x38100000, 0x38102000, 0x38104000, - 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000, - 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000, - 0x3811A000, 0x3811C000, 0x3811E000, 0x38120000, 0x38122000, - 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, - 0x3812E000, 0x38130000, 0x38132000, 0x38134000, 0x38136000, - 0x38138000, 0x3813A000, 0x3813C000, 0x3813E000, 0x38140000, - 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000, - 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000, - 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000, - 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, - 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, - 0x38174000, 0x38176000, 0x38178000, 0x3817A000, 0x3817C000, - 0x3817E000, 0x38180000, 0x38182000, 0x38184000, 0x38186000, - 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000, - 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, - 0x3819C000, 0x3819E000, 0x381A0000, 0x381A2000, 0x381A4000, - 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, - 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000, - 0x381BA000, 0x381BC000, 0x381BE000, 0x381C0000, 0x381C2000, - 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, 0x381CC000, - 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, - 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000, 0x381E0000, - 0x381E2000, 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, - 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, 0x381F4000, - 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000, - 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000, - 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, - 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, - 0x3821E000, 0x38220000, 0x38222000, 0x38224000, 0x38226000, - 0x38228000, 0x3822A000, 0x3822C000, 0x3822E000, 0x38230000, - 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823A000, - 0x3823C000, 0x3823E000, 0x38240000, 0x38242000, 0x38244000, - 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, - 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, - 0x3825A000, 0x3825C000, 0x3825E000, 0x38260000, 0x38262000, - 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000, - 0x3826E000, 0x38270000, 0x38272000, 0x38274000, 0x38276000, - 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000, 0x38280000, - 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, - 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, - 0x38296000, 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000, - 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000, - 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000, - 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, 0x382BC000, - 0x382BE000, 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, - 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, - 0x382D2000, 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, - 0x382DC000, 0x382DE000, 0x382E0000, 0x382E2000, 0x382E4000, - 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000, - 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000, - 0x382FA000, 0x382FC000, 0x382FE000, 0x38300000, 0x38302000, - 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, - 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, - 0x38318000, 0x3831A000, 0x3831C000, 0x3831E000, 0x38320000, - 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832A000, - 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000, - 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000, - 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, - 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, - 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000, - 0x3835E000, 0x38360000, 0x38362000, 0x38364000, 0x38366000, - 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, 0x38370000, - 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, - 0x3837C000, 0x3837E000, 0x38380000, 0x38382000, 0x38384000, - 0x38386000, 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, - 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000, - 0x3839A000, 0x3839C000, 0x3839E000, 0x383A0000, 0x383A2000, - 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000, - 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, - 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000, 0x383C0000, - 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, - 0x383CC000, 0x383CE000, 0x383D0000, 0x383D2000, 0x383D4000, - 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, 0x383DE000, - 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000, - 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, - 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, - 0x383FE000, 0x38400000, 0x38402000, 0x38404000, 0x38406000, - 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000, - 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841A000, - 0x3841C000, 0x3841E000, 0x38420000, 0x38422000, 0x38424000, - 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, - 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, - 0x3843A000, 0x3843C000, 0x3843E000, 0x38440000, 0x38442000, - 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000, - 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, - 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000, 0x38460000, - 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, - 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, - 0x38476000, 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000, - 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000, - 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000, - 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000, - 0x3849E000, 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, - 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, - 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, - 0x384BC000, 0x384BE000, 0x384C0000, 0x384C2000, 0x384C4000, - 0x384C6000, 0x384C8000, 0x384CA000, 0x384CC000, 0x384CE000, - 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000, - 0x384DA000, 0x384DC000, 0x384DE000, 0x384E0000, 0x384E2000, - 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, - 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, - 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000, 0x38500000, - 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850A000, - 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, 0x38514000, - 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000, - 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, - 0x3852A000, 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, - 0x38534000, 0x38536000, 0x38538000, 0x3853A000, 0x3853C000, - 0x3853E000, 0x38540000, 0x38542000, 0x38544000, 0x38546000, - 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000, - 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, - 0x3855C000, 0x3855E000, 0x38560000, 0x38562000, 0x38564000, - 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, - 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000, - 0x3857A000, 0x3857C000, 0x3857E000, 0x38580000, 0x38582000, - 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000, - 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, - 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000, 0x385A0000, - 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, - 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000, - 0x385B6000, 0x385B8000, 0x385BA000, 0x385BC000, 0x385BE000, - 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, 0x385C8000, - 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, - 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, - 0x385DE000, 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, - 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000, - 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000, - 0x385FC000, 0x385FE000, 0x38600000, 0x38602000, 0x38604000, - 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, - 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, - 0x3861A000, 0x3861C000, 0x3861E000, 0x38620000, 0x38622000, - 0x38624000, 0x38626000, 0x38628000, 0x3862A000, 0x3862C000, - 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, - 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000, 0x38640000, - 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, - 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, - 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000, - 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000, - 0x3866A000, 0x3866C000, 0x3866E000, 0x38670000, 0x38672000, - 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000, - 0x3867E000, 0x38680000, 0x38682000, 0x38684000, 0x38686000, - 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, - 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869A000, - 0x3869C000, 0x3869E000, 0x386A0000, 0x386A2000, 0x386A4000, - 0x386A6000, 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000, - 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, 0x386B8000, - 0x386BA000, 0x386BC000, 0x386BE000, 0x386C0000, 0x386C2000, - 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, - 0x386CE000, 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, - 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000, 0x386E0000, - 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000, - 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000, - 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000, - 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, - 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, - 0x38714000, 0x38716000, 0x38718000, 0x3871A000, 0x3871C000, - 0x3871E000, 0x38720000, 0x38722000, 0x38724000, 0x38726000, - 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000, - 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, - 0x3873C000, 0x3873E000, 0x38740000, 0x38742000, 0x38744000, - 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, - 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, - 0x3875A000, 0x3875C000, 0x3875E000, 0x38760000, 0x38762000, - 0x38764000, 0x38766000, 0x38768000, 0x3876A000, 0x3876C000, - 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, - 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000, 0x38780000, - 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878A000, - 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000, - 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000, - 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, 0x387A8000, - 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, - 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, - 0x387BE000, 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, - 0x387C8000, 0x387CA000, 0x387CC000, 0x387CE000, 0x387D0000, - 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000, - 0x387DC000, 0x387DE000, 0x387E0000, 0x387E2000, 0x387E4000, - 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, - 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, - 0x387FA000, 0x387FC000, 0x387FE000}; - static const uint32 exponent_table[64] = { - 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, - 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, - 0x06000000, 0x06800000, 0x07000000, 0x07800000, 0x08000000, 0x08800000, - 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, - 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, - 0x0F000000, 0x47800000, 0x80000000, 0x80800000, 0x81000000, 0x81800000, - 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, - 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000, - 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, - 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, - 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000}; - static const unsigned short offset_table[64] = { - 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, - 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, - 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 0, - 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, - 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, - 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024}; - uint32 bits = mantissa_table[offset_table[value >> 10] + (value & 0x3FF)] + - exponent_table[value >> 10]; - // return *reinterpret_cast(&bits); //violating - //strict aliasing! - float out; - std::memcpy(&out, &bits, sizeof(float)); - return out; -} -/// Convert half-precision to IEEE double-precision. -/// \param value binary representation of half-precision value -/// \return double-precision value -inline double half2float_impl(uint16 value, double, true_type) { - typedef bits::type uint32; - typedef bits::type uint64; - uint32 hi = static_cast(value & 0x8000) << 16; - int abs = value & 0x7FFF; - if (abs) { - hi |= 0x3F000000 << static_cast(abs >= 0x7C00); - for (; abs < 0x400; abs <<= 1, hi -= 0x100000) - ; - hi += static_cast(abs) << 10; - } - uint64 bits = static_cast(hi) << 32; - // return *reinterpret_cast(&bits); //violating - //strict aliasing! - double out; - std::memcpy(&out, &bits, sizeof(double)); - return out; -} + /// Binary exponential implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr exp2(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::exp2(arg)); + #else + return expr(static_cast(std::exp(arg*0.69314718055994530941723212145818))); + #endif + } -/// Convert half-precision to non-IEEE floating point. -/// \tparam T type to convert to (builtin integer type) -/// \param value binary representation of half-precision value -/// \return floating point value -template -T half2float_impl(uint16 value, T, ...) { - T out; - int abs = value & 0x7FFF; - if (abs > 0x7C00) - out = std::numeric_limits::has_quiet_NaN - ? std::numeric_limits::quiet_NaN() - : T(); - else if (abs == 0x7C00) - out = std::numeric_limits::has_infinity - ? std::numeric_limits::infinity() - : std::numeric_limits::max(); - else if (abs > 0x3FF) - out = std::ldexp(static_cast((abs & 0x3FF) | 0x400), (abs >> 10) - 25); - else - out = std::ldexp(static_cast(abs), -24); - return (value & 0x8000) ? -out : out; -} + /// Logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log(float arg) { return expr(std::log(arg)); } -/// Convert half-precision to floating point. -/// \tparam T type to convert to (builtin integer type) -/// \param value binary representation of half-precision value -/// \return floating point value -template -T half2float(uint16 value) { - return half2float_impl(value, T(), - bool_type < std::numeric_limits::is_iec559 && - sizeof(typename bits::type) == sizeof(T) > ()); -} + /// Common logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log10(float arg) { return expr(std::log10(arg)); } -/// Convert half-precision floating point to integer. -/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest -/// rounding -/// \tparam E `true` for round to even, `false` for round away from zero -/// \tparam T type to convert to (buitlin integer type with at least 16 bits -/// precision, excluding any implicit sign bits) -/// \param value binary representation of half-precision value -/// \return integral value -template -T half2int_impl(uint16 value) { -#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_integral::value, - "half to int conversion only supports builtin integer types"); -#endif - unsigned int e = value & 0x7FFF; - if (e >= 0x7C00) - return (value & 0x8000) ? std::numeric_limits::min() - : std::numeric_limits::max(); - if (e < 0x3800) { - if (R == std::round_toward_infinity) - return T(~(value >> 15) & (e != 0)); - else if (R == std::round_toward_neg_infinity) - return -T(value > 0x8000); - return T(); - } - unsigned int m = (value & 0x3FF) | 0x400; - e >>= 10; - if (e < 25) { - if (R == std::round_to_nearest) - m += (1 << (24 - e)) - (~(m >> (25 - e)) & E); - else if (R == std::round_toward_infinity) - m += ((value >> 15) - 1) & ((1 << (25 - e)) - 1U); - else if (R == std::round_toward_neg_infinity) - m += -(value >> 15) & ((1 << (25 - e)) - 1U); - m >>= 25 - e; - } else - m <<= e - 25; - return (value & 0x8000) ? -static_cast(m) : static_cast(m); -} + /// Logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log1p(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::log1p(arg)); + #else + return expr(static_cast(std::log(1.0+arg))); + #endif + } -/// Convert half-precision floating point to integer. -/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest -/// rounding -/// \tparam T type to convert to (buitlin integer type with at least 16 bits -/// precision, excluding any implicit sign bits) -/// \param value binary representation of half-precision value -/// \return integral value -template -T half2int(uint16 value) { - return half2int_impl(value); -} + /// Binary logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log2(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::log2(arg)); + #else + return expr(static_cast(std::log(static_cast(arg))*1.4426950408889634073599246810019)); + #endif + } -/// Convert half-precision floating point to integer using -/// round-to-nearest-away-from-zero. -/// \tparam T type to convert to (buitlin integer type with at least 16 bits -/// precision, excluding any implicit sign bits) -/// \param value binary representation of half-precision value -/// \return integral value -template -T half2int_up(uint16 value) { - return half2int_impl(value); -} + /// Square root implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr sqrt(float arg) { return expr(std::sqrt(arg)); } -/// Round half-precision number to nearest integer value. -/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest -/// rounding -/// \tparam E `true` for round to even, `false` for round away from zero -/// \param value binary representation of half-precision value -/// \return half-precision bits for nearest integral value -template -uint16 round_half_impl(uint16 value) { - unsigned int e = value & 0x7FFF; - uint16 result = value; - if (e < 0x3C00) { - result &= 0x8000; - if (R == std::round_to_nearest) - result |= 0x3C00U & -(e >= (0x3800 + E)); - else if (R == std::round_toward_infinity) - result |= 0x3C00U & -(~(value >> 15) & (e != 0)); - else if (R == std::round_toward_neg_infinity) - result |= 0x3C00U & -(value > 0x8000); - } else if (e < 0x6400) { - e = 25 - (e >> 10); - unsigned int mask = (1 << e) - 1; - if (R == std::round_to_nearest) - result += (1 << (e - 1)) - (~(result >> e) & E); - else if (R == std::round_toward_infinity) - result += mask & ((value >> 15) - 1); - else if (R == std::round_toward_neg_infinity) - result += mask & -(value >> 15); - result &= ~mask; - } - return result; -} + /// Cubic root implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr cbrt(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::cbrt(arg)); + #else + if(builtin_isnan(arg) || builtin_isinf(arg)) + return expr(arg); + return expr(builtin_signbit(arg) ? -static_cast(std::pow(-static_cast(arg), 1.0/3.0)) : + static_cast(std::pow(static_cast(arg), 1.0/3.0))); + #endif + } -/// Round half-precision number to nearest integer value. -/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest -/// rounding -/// \param value binary representation of half-precision value -/// \return half-precision bits for nearest integral value -template -uint16 round_half(uint16 value) { - return round_half_impl(value); -} + /// Hypotenuse implementation. + /// \param x first argument + /// \param y second argument + /// \return function value stored in single-preicision + static expr hypot(float x, float y) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::hypot(x, y)); + #else + return expr((builtin_isinf(x) || builtin_isinf(y)) ? std::numeric_limits::infinity() : + static_cast(std::sqrt(static_cast(x)*x+static_cast(y)*y))); + #endif + } -/// Round half-precision number to nearest integer value using -/// round-to-nearest-away-from-zero. -/// \param value binary representation of half-precision value -/// \return half-precision bits for nearest integral value -inline uint16 round_half_up(uint16 value) { - return round_half_impl(value); -} -/// \} - -struct functions; -template -struct unary_specialized; -template -struct binary_specialized; -template -struct half_caster; -} + /// Power implementation. + /// \param base value to exponentiate + /// \param exp power to expontiate to + /// \return function value stored in single-preicision + static expr pow(float base, float exp) { return expr(std::pow(base, exp)); } + + /// Sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr sin(float arg) { return expr(std::sin(arg)); } + + /// Cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr cos(float arg) { return expr(std::cos(arg)); } + + /// Tan implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr tan(float arg) { return expr(std::tan(arg)); } + + /// Arc sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr asin(float arg) { return expr(std::asin(arg)); } + + /// Arc cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr acos(float arg) { return expr(std::acos(arg)); } + + /// Arc tangent implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr atan(float arg) { return expr(std::atan(arg)); } + + /// Arc tangent implementation. + /// \param x first argument + /// \param y second argument + /// \return function value stored in single-preicision + static expr atan2(float x, float y) { return expr(std::atan2(x, y)); } + + /// Hyperbolic sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr sinh(float arg) { return expr(std::sinh(arg)); } + + /// Hyperbolic cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr cosh(float arg) { return expr(std::cosh(arg)); } + + /// Hyperbolic tangent implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr tanh(float arg) { return expr(std::tanh(arg)); } + + /// Hyperbolic area sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr asinh(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::asinh(arg)); + #else + return expr((arg==-std::numeric_limits::infinity()) ? arg : static_cast(std::log(arg+std::sqrt(arg*arg+1.0)))); + #endif + } -/// Half-precision floating point type. -/// This class implements an IEEE-conformant half-precision floating point type -/// with the usual arithmetic operators and -/// conversions. It is implicitly convertible to single-precision floating -/// point, which makes artihmetic expressions and -/// functions with mixed-type operands to be of the most precise operand type. -/// Additionally all arithmetic operations -/// (and many mathematical functions) are carried out in single-precision -/// internally. All conversions from single- to -/// half-precision are done using the library's default rounding mode, but -/// temporary results inside chained arithmetic -/// expressions are kept in single-precision as long as possible (while of -/// course still maintaining a strong half-precision type). -/// -/// According to the C++98/03 definition, the half type is not a POD type. But -/// according to C++11's less strict and -/// extended definitions it is both a standard layout type and a trivially -/// copyable type (even if not a POD type), which -/// means it can be standard-conformantly copied using raw binary copies. But in -/// this context some more words about the -/// actual size of the type. Although the half is representing an IEEE 16-bit -/// type, it does not neccessarily have to be of -/// exactly 16-bits size. But on any reasonable implementation the actual binary -/// representation of this type will most -/// probably not ivolve any additional "magic" or padding beyond the simple -/// binary representation of the underlying 16-bit -/// IEEE number, even if not strictly guaranteed by the standard. But even then -/// it only has an actual size of 16 bits if -/// your C++ implementation supports an unsigned integer type of exactly 16 bits -/// width. But this should be the case on -/// nearly any reasonable platform. -/// -/// So if your C++ implementation is not totally exotic or imposes special -/// alignment requirements, it is a reasonable -/// assumption that the data of a half is just comprised of the 2 bytes of the -/// underlying IEEE representation. -class half { - friend struct detail::functions; - friend struct detail::unary_specialized; - friend struct detail::binary_specialized; - template - friend struct detail::half_caster; - friend class std::numeric_limits; -#if HALF_ENABLE_CPP11_HASH - friend struct std::hash; -#endif -#if HALF_ENABLE_CPP11_USER_LITERALS - friend half literal::operator"" _h(long double); -#endif + /// Hyperbolic area cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr acosh(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::acosh(arg)); + #else + return expr((arg<-1.0f) ? std::numeric_limits::quiet_NaN() : static_cast(std::log(arg+std::sqrt(arg*arg-1.0)))); + #endif + } - public: - /// Default constructor. - /// This initializes the half to 0. Although this does not match the builtin - /// types' default-initialization semantics - /// and may be less efficient than no initialization, it is needed to provide - /// proper value-initialization semantics. - HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {} - - /// Copy constructor. - /// \tparam T type of concrete half expression - /// \param rhs half expression to copy from - half(detail::expr rhs) - : data_(detail::float2half(static_cast(rhs))) {} - - /// Conversion constructor. - /// \param rhs float to convert - explicit half(float rhs) : data_(detail::float2half(rhs)) {} - - /// Conversion to single-precision. - /// \return single precision value representing expression value - operator float() const { return detail::half2float(data_); } - - /// Assignment operator. - /// \tparam T type of concrete half expression - /// \param rhs half expression to copy from - /// \return reference to this half - half &operator=(detail::expr rhs) { return *this = static_cast(rhs); } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to add - /// \return reference to this half - template - typename detail::enable::type operator+=(T rhs) { - return *this += static_cast(rhs); - } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to subtract - /// \return reference to this half - template - typename detail::enable::type operator-=(T rhs) { - return *this -= static_cast(rhs); - } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to multiply with - /// \return reference to this half - template - typename detail::enable::type operator*=(T rhs) { - return *this *= static_cast(rhs); - } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to divide by - /// \return reference to this half - template - typename detail::enable::type operator/=(T rhs) { - return *this /= static_cast(rhs); - } - - /// Assignment operator. - /// \param rhs single-precision value to copy from - /// \return reference to this half - half &operator=(float rhs) { - data_ = detail::float2half(rhs); - return *this; - } - - /// Arithmetic assignment. - /// \param rhs single-precision value to add - /// \return reference to this half - half &operator+=(float rhs) { - data_ = - detail::float2half(detail::half2float(data_) + rhs); - return *this; - } - - /// Arithmetic assignment. - /// \param rhs single-precision value to subtract - /// \return reference to this half - half &operator-=(float rhs) { - data_ = - detail::float2half(detail::half2float(data_) - rhs); - return *this; - } - - /// Arithmetic assignment. - /// \param rhs single-precision value to multiply with - /// \return reference to this half - half &operator*=(float rhs) { - data_ = - detail::float2half(detail::half2float(data_) * rhs); - return *this; - } - - /// Arithmetic assignment. - /// \param rhs single-precision value to divide by - /// \return reference to this half - half &operator/=(float rhs) { - data_ = - detail::float2half(detail::half2float(data_) / rhs); - return *this; - } - - /// Prefix increment. - /// \return incremented half value - half &operator++() { return *this += 1.0f; } - - /// Prefix decrement. - /// \return decremented half value - half &operator--() { return *this -= 1.0f; } - - /// Postfix increment. - /// \return non-incremented half value - half operator++(int) { - half out(*this); - ++*this; - return out; - } - - /// Postfix decrement. - /// \return non-decremented half value - half operator--(int) { - half out(*this); - --*this; - return out; - } - - private: - /// Rounding mode to use - static const std::float_round_style round_style = - (std::float_round_style)(HALF_ROUND_STYLE); - - /// Constructor. - /// \param bits binary representation to set half to - HALF_CONSTEXPR half(detail::binary_t, detail::uint16 bits) HALF_NOEXCEPT - : data_(bits) {} - - /// Internal binary representation - detail::uint16 data_; -}; + /// Hyperbolic area tangent implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr atanh(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::atanh(arg)); + #else + return expr(static_cast(0.5*std::log((1.0+arg)/(1.0-arg)))); + #endif + } -#if HALF_ENABLE_CPP11_USER_LITERALS -namespace literal { -/// Half literal. -/// While this returns an actual half-precision value, half literals can -/// unfortunately not be constant expressions due -/// to rather involved conversions. -/// \param value literal value -/// \return half with given value (if representable) -inline half operator"" _h(long double value) { - return half(detail::binary, detail::float2half(value)); -} -} -#endif + /// Error function implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr erf(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::erf(arg)); + #else + return expr(static_cast(erf(static_cast(arg)))); + #endif + } -namespace detail { -/// Wrapper implementing unspecialized half-precision functions. -struct functions { - /// Addition implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision sum stored in single-precision - static expr plus(float x, float y) { return expr(x + y); } - - /// Subtraction implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision difference stored in single-precision - static expr minus(float x, float y) { return expr(x - y); } - - /// Multiplication implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision product stored in single-precision - static expr multiplies(float x, float y) { return expr(x * y); } - - /// Division implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision quotient stored in single-precision - static expr divides(float x, float y) { return expr(x / y); } - - /// Output implementation. - /// \param out stream to write to - /// \param arg value to write - /// \return reference to stream - template - static std::basic_ostream &write( - std::basic_ostream &out, float arg) { - return out << arg; - } - - /// Input implementation. - /// \param in stream to read from - /// \param arg half to read into - /// \return reference to stream - template - static std::basic_istream &read( - std::basic_istream &in, half &arg) { - float f; - if (in >> f) arg = f; - return in; - } - - /// Modulo implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision division remainder stored in single-precision - static expr fmod(float x, float y) { return expr(std::fmod(x, y)); } - - /// Remainder implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision division remainder stored in single-precision - static expr remainder(float x, float y) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::remainder(x, y)); -#else - if (builtin_isnan(x) || builtin_isnan(y)) - return expr(std::numeric_limits::quiet_NaN()); - float ax = std::fabs(x), ay = std::fabs(y); - if (ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) - return expr(std::numeric_limits::quiet_NaN()); - if (ay >= 65536.0f) return expr(x); - if (ax == ay) return expr(builtin_signbit(x) ? -0.0f : 0.0f); - ax = std::fmod(ax, ay + ay); - float y2 = 0.5f * ay; - if (ax > y2) { - ax -= ay; - if (ax >= y2) ax -= ay; - } - return expr(builtin_signbit(x) ? -ax : ax); -#endif - } - - /// Remainder implementation. - /// \param x first operand - /// \param y second operand - /// \param quo address to store quotient bits at - /// \return Half-precision division remainder stored in single-precision - static expr remquo(float x, float y, int *quo) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::remquo(x, y, quo)); -#else - if (builtin_isnan(x) || builtin_isnan(y)) - return expr(std::numeric_limits::quiet_NaN()); - bool sign = builtin_signbit(x), - qsign = static_cast(sign ^ builtin_signbit(y)); - float ax = std::fabs(x), ay = std::fabs(y); - if (ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) - return expr(std::numeric_limits::quiet_NaN()); - if (ay >= 65536.0f) return expr(x); - if (ax == ay) return *quo = qsign ? -1 : 1, expr(sign ? -0.0f : 0.0f); - ax = std::fmod(ax, 8.0f * ay); - int cquo = 0; - if (ax >= 4.0f * ay) { - ax -= 4.0f * ay; - cquo += 4; - } - if (ax >= 2.0f * ay) { - ax -= 2.0f * ay; - cquo += 2; - } - float y2 = 0.5f * ay; - if (ax > y2) { - ax -= ay; - ++cquo; - if (ax >= y2) { - ax -= ay; - ++cquo; - } - } - return *quo = qsign ? -cquo : cquo, expr(sign ? -ax : ax); -#endif - } - - /// Positive difference implementation. - /// \param x first operand - /// \param y second operand - /// \return Positive difference stored in single-precision - static expr fdim(float x, float y) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::fdim(x, y)); -#else - return expr((x <= y) ? 0.0f : (x - y)); -#endif - } - - /// Fused multiply-add implementation. - /// \param x first operand - /// \param y second operand - /// \param z third operand - /// \return \a x * \a y + \a z stored in single-precision - static expr fma(float x, float y, float z) { -#if HALF_ENABLE_CPP11_CMATH && defined(FP_FAST_FMAF) - return expr(std::fma(x, y, z)); -#else - return expr(x * y + z); -#endif - } - - /// Get NaN. - /// \return Half-precision quiet NaN - static half nanh() { return half(binary, 0x7FFF); } - - /// Exponential implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr exp(float arg) { return expr(std::exp(arg)); } - - /// Exponential implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr expm1(float arg) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::expm1(arg)); -#else - return expr(static_cast(std::exp(static_cast(arg)) - 1.0)); -#endif - } - - /// Binary exponential implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr exp2(float arg) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::exp2(arg)); -#else - return expr( - static_cast(std::exp(arg * 0.69314718055994530941723212145818))); -#endif - } - - /// Logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr log(float arg) { return expr(std::log(arg)); } - - /// Common logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr log10(float arg) { return expr(std::log10(arg)); } - - /// Logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr log1p(float arg) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::log1p(arg)); -#else - return expr(static_cast(std::log(1.0 + arg))); -#endif - } - - /// Binary logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr log2(float arg) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::log2(arg)); -#else - return expr(static_cast(std::log(static_cast(arg)) * - 1.4426950408889634073599246810019)); -#endif - } - - /// Square root implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr sqrt(float arg) { return expr(std::sqrt(arg)); } - - /// Cubic root implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr cbrt(float arg) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::cbrt(arg)); -#else - if (builtin_isnan(arg) || builtin_isinf(arg)) return expr(arg); - return expr(builtin_signbit(arg) - ? -static_cast( - std::pow(-static_cast(arg), 1.0 / 3.0)) - : static_cast( - std::pow(static_cast(arg), 1.0 / 3.0))); -#endif - } - - /// Hypotenuse implementation. - /// \param x first argument - /// \param y second argument - /// \return function value stored in single-preicision - static expr hypot(float x, float y) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::hypot(x, y)); -#else - return expr( - (builtin_isinf(x) || builtin_isinf(y)) - ? std::numeric_limits::infinity() - : static_cast(std::sqrt(static_cast(x) * x + - static_cast(y) * y))); -#endif - } - - /// Power implementation. - /// \param base value to exponentiate - /// \param exp power to expontiate to - /// \return function value stored in single-preicision - static expr pow(float base, float exp) { return expr(std::pow(base, exp)); } - - /// Sine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr sin(float arg) { return expr(std::sin(arg)); } - - /// Cosine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr cos(float arg) { return expr(std::cos(arg)); } - - /// Tan implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr tan(float arg) { return expr(std::tan(arg)); } - - /// Arc sine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr asin(float arg) { return expr(std::asin(arg)); } - - /// Arc cosine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr acos(float arg) { return expr(std::acos(arg)); } - - /// Arc tangent implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr atan(float arg) { return expr(std::atan(arg)); } - - /// Arc tangent implementation. - /// \param x first argument - /// \param y second argument - /// \return function value stored in single-preicision - static expr atan2(float x, float y) { return expr(std::atan2(x, y)); } - - /// Hyperbolic sine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr sinh(float arg) { return expr(std::sinh(arg)); } - - /// Hyperbolic cosine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr cosh(float arg) { return expr(std::cosh(arg)); } - - /// Hyperbolic tangent implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr tanh(float arg) { return expr(std::tanh(arg)); } - - /// Hyperbolic area sine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr asinh(float arg) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::asinh(arg)); -#else - return expr( - (arg == -std::numeric_limits::infinity()) - ? arg - : static_cast(std::log(arg + std::sqrt(arg * arg + 1.0)))); -#endif - } - - /// Hyperbolic area cosine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr acosh(float arg) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::acosh(arg)); -#else - return expr((arg < -1.0f) ? std::numeric_limits::quiet_NaN() - : static_cast(std::log( - arg + std::sqrt(arg * arg - 1.0)))); -#endif - } - - /// Hyperbolic area tangent implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr atanh(float arg) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::atanh(arg)); -#else - return expr(static_cast(0.5 * std::log((1.0 + arg) / (1.0 - arg)))); -#endif - } - - /// Error function implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr erf(float arg) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::erf(arg)); -#else - return expr(static_cast(erf(static_cast(arg)))); -#endif - } - - /// Complementary implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr erfc(float arg) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::erfc(arg)); -#else - return expr(static_cast(1.0 - erf(static_cast(arg)))); -#endif - } - - /// Gamma logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr lgamma(float arg) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::lgamma(arg)); -#else - if (builtin_isinf(arg)) return expr(std::numeric_limits::infinity()); - if (arg < 0.0f) { - float i, f = std::modf(-arg, &i); - if (f == 0.0f) return expr(std::numeric_limits::infinity()); - return expr(static_cast( - 1.1447298858494001741434273513531 - - std::log(std::abs(std::sin(3.1415926535897932384626433832795 * f))) - - lgamma(1.0 - arg))); - } - return expr(static_cast(lgamma(static_cast(arg)))); -#endif - } - - /// Gamma implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr tgamma(float arg) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::tgamma(arg)); -#else - if (arg == 0.0f) - return builtin_signbit(arg) - ? expr(-std::numeric_limits::infinity()) - : expr(std::numeric_limits::infinity()); - if (arg < 0.0f) { - float i, f = std::modf(-arg, &i); - if (f == 0.0f) return expr(std::numeric_limits::quiet_NaN()); - double value = 3.1415926535897932384626433832795 / - (std::sin(3.1415926535897932384626433832795 * f) * - std::exp(lgamma(1.0 - arg))); - return expr( - static_cast((std::fmod(i, 2.0f) == 0.0f) ? -value : value)); - } - if (builtin_isinf(arg)) return expr(arg); - return expr(static_cast(std::exp(lgamma(static_cast(arg))))); -#endif - } - - /// Floor implementation. - /// \param arg value to round - /// \return rounded value - static half floor(half arg) { - return half(binary, round_half(arg.data_)); - } - - /// Ceiling implementation. - /// \param arg value to round - /// \return rounded value - static half ceil(half arg) { - return half(binary, round_half(arg.data_)); - } - - /// Truncation implementation. - /// \param arg value to round - /// \return rounded value - static half trunc(half arg) { - return half(binary, round_half(arg.data_)); - } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static half round(half arg) { return half(binary, round_half_up(arg.data_)); } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static long lround(half arg) { return detail::half2int_up(arg.data_); } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static half rint(half arg) { - return half(binary, round_half(arg.data_)); - } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static long lrint(half arg) { - return detail::half2int(arg.data_); - } + /// Complementary implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr erfc(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::erfc(arg)); + #else + return expr(static_cast(1.0-erf(static_cast(arg)))); + #endif + } -#if HALF_ENABLE_CPP11_LONG_LONG - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static long long llround(half arg) { - return detail::half2int_up(arg.data_); - } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static long long llrint(half arg) { - return detail::half2int(arg.data_); - } -#endif + /// Gamma logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr lgamma(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::lgamma(arg)); + #else + if(builtin_isinf(arg)) + return expr(std::numeric_limits::infinity()); + if(arg < 0.0f) + { + float i, f = std::modf(-arg, &i); + if(f == 0.0f) + return expr(std::numeric_limits::infinity()); + return expr(static_cast(1.1447298858494001741434273513531- + std::log(std::abs(std::sin(3.1415926535897932384626433832795*f)))-lgamma(1.0-arg))); + } + return expr(static_cast(lgamma(static_cast(arg)))); + #endif + } - /// Decompression implementation. - /// \param arg number to decompress - /// \param exp address to store exponent at - /// \return normalized significant - static half frexp(half arg, int *exp) { - int m = arg.data_ & 0x7FFF, e = -14; - if (m >= 0x7C00 || !m) return *exp = 0, arg; - for (; m < 0x400; m <<= 1, --e) - ; - return *exp = e + (m >> 10), - half(binary, (arg.data_ & 0x8000) | 0x3800 | (m & 0x3FF)); - } - - /// Decompression implementation. - /// \param arg number to decompress - /// \param iptr address to store integer part at - /// \return fractional part - static half modf(half arg, half *iptr) { - unsigned int e = arg.data_ & 0x7FFF; - if (e >= 0x6400) - return *iptr = arg, half(binary, arg.data_ & (0x8000U | -(e > 0x7C00))); - if (e < 0x3C00) return iptr->data_ = arg.data_ & 0x8000, arg; - e >>= 10; - unsigned int mask = (1 << (25 - e)) - 1, m = arg.data_ & mask; - iptr->data_ = arg.data_ & ~mask; - if (!m) return half(binary, arg.data_ & 0x8000); - for (; m < 0x400; m <<= 1, --e) - ; - return half(binary, static_cast((arg.data_ & 0x8000) | (e << 10) | - (m & 0x3FF))); - } - - /// Scaling implementation. - /// \param arg number to scale - /// \param exp power of two to scale by - /// \return scaled number - static half scalbln(half arg, long exp) { - unsigned int m = arg.data_ & 0x7FFF; - if (m >= 0x7C00 || !m) return arg; - for (; m < 0x400; m <<= 1, --exp) - ; - exp += m >> 10; - uint16 value = arg.data_ & 0x8000; - if (exp > 30) { - if (half::round_style == std::round_toward_zero) - value |= 0x7BFF; - else if (half::round_style == std::round_toward_infinity) - value |= 0x7C00 - (value >> 15); - else if (half::round_style == std::round_toward_neg_infinity) - value |= 0x7BFF + (value >> 15); - else - value |= 0x7C00; - } else if (exp > 0) - value |= (exp << 10) | (m & 0x3FF); - else if (exp > -11) { - m = (m & 0x3FF) | 0x400; - if (half::round_style == std::round_to_nearest) { - m += 1 << -exp; -#if HALF_ROUND_TIES_TO_EVEN - m -= (m >> (1 - exp)) & 1; -#endif - } else if (half::round_style == std::round_toward_infinity) - m += ((value >> 15) - 1) & ((1 << (1 - exp)) - 1U); - else if (half::round_style == std::round_toward_neg_infinity) - m += -(value >> 15) & ((1 << (1 - exp)) - 1U); - value |= m >> (1 - exp); - } else if (half::round_style == std::round_toward_infinity) - value -= (value >> 15) - 1; - else if (half::round_style == std::round_toward_neg_infinity) - value += value >> 15; - return half(binary, value); - } - - /// Exponent implementation. - /// \param arg number to query - /// \return floating point exponent - static int ilogb(half arg) { - int abs = arg.data_ & 0x7FFF; - if (!abs) return FP_ILOGB0; - if (abs < 0x7C00) { - int exp = (abs >> 10) - 15; - if (abs < 0x400) - for (; abs < 0x200; abs <<= 1, --exp) - ; - return exp; - } - if (abs > 0x7C00) return FP_ILOGBNAN; - return INT_MAX; - } - - /// Exponent implementation. - /// \param arg number to query - /// \return floating point exponent - static half logb(half arg) { - int abs = arg.data_ & 0x7FFF; - if (!abs) return half(binary, 0xFC00); - if (abs < 0x7C00) { - int exp = (abs >> 10) - 15; - if (abs < 0x400) - for (; abs < 0x200; abs <<= 1, --exp) - ; - uint16 bits = (exp < 0) << 15; - if (exp) { - unsigned int m = std::abs(exp) << 6, e = 18; - for (; m < 0x400; m <<= 1, --e) - ; - bits |= (e << 10) + m; - } - return half(binary, bits); - } - if (abs > 0x7C00) return arg; - return half(binary, 0x7C00); - } - - /// Enumeration implementation. - /// \param from number to increase/decrease - /// \param to direction to enumerate into - /// \return next representable number - static half nextafter(half from, half to) { - uint16 fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF; - if (fabs > 0x7C00) return from; - if (tabs > 0x7C00 || from.data_ == to.data_ || !(fabs | tabs)) return to; - if (!fabs) return half(binary, (to.data_ & 0x8000) + 1); - bool lt = - ((fabs == from.data_) ? static_cast(fabs) - : -static_cast(fabs)) < - ((tabs == to.data_) ? static_cast(tabs) : -static_cast(tabs)); - return half(binary, - from.data_ + - (((from.data_ >> 15) ^ static_cast(lt)) << 1) - - 1); - } - - /// Enumeration implementation. - /// \param from number to increase/decrease - /// \param to direction to enumerate into - /// \return next representable number - static half nexttoward(half from, long double to) { - if (isnan(from)) return from; - long double lfrom = static_cast(from); - if (builtin_isnan(to) || lfrom == to) return half(static_cast(to)); - if (!(from.data_ & 0x7FFF)) - return half(binary, - (static_cast(builtin_signbit(to)) << 15) + 1); - return half( - binary, - from.data_ + - (((from.data_ >> 15) ^ static_cast(lfrom < to)) << 1) - - 1); - } - - /// Sign implementation - /// \param x first operand - /// \param y second operand - /// \return composed value - static half copysign(half x, half y) { - return half(binary, x.data_ ^ ((x.data_ ^ y.data_) & 0x8000)); - } - - /// Classification implementation. - /// \param arg value to classify - /// \retval true if infinite number - /// \retval false else - static int fpclassify(half arg) { - unsigned int abs = arg.data_ & 0x7FFF; - return abs ? ((abs > 0x3FF) ? ((abs >= 0x7C00) - ? ((abs > 0x7C00) ? FP_NAN : FP_INFINITE) - : FP_NORMAL) - : FP_SUBNORMAL) - : FP_ZERO; - } - - /// Classification implementation. - /// \param arg value to classify - /// \retval true if finite number - /// \retval false else - static bool isfinite(half arg) { return (arg.data_ & 0x7C00) != 0x7C00; } - - /// Classification implementation. - /// \param arg value to classify - /// \retval true if infinite number - /// \retval false else - static bool isinf(half arg) { return (arg.data_ & 0x7FFF) == 0x7C00; } - - /// Classification implementation. - /// \param arg value to classify - /// \retval true if not a number - /// \retval false else - static bool isnan(half arg) { return (arg.data_ & 0x7FFF) > 0x7C00; } - - /// Classification implementation. - /// \param arg value to classify - /// \retval true if normal number - /// \retval false else - static bool isnormal(half arg) { - return ((arg.data_ & 0x7C00) != 0) & ((arg.data_ & 0x7C00) != 0x7C00); - } - - /// Sign bit implementation. - /// \param arg value to check - /// \retval true if signed - /// \retval false if unsigned - static bool signbit(half arg) { return (arg.data_ & 0x8000) != 0; } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if operands equal - /// \retval false else - static bool isequal(half x, half y) { - return (x.data_ == y.data_ || !((x.data_ | y.data_) & 0x7FFF)) && !isnan(x); - } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if operands not equal - /// \retval false else - static bool isnotequal(half x, half y) { - return (x.data_ != y.data_ && ((x.data_ | y.data_) & 0x7FFF)) || isnan(x); - } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x > \a y - /// \retval false else - static bool isgreater(half x, half y) { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - return xabs <= 0x7C00 && yabs <= 0x7C00 && - (((xabs == x.data_) ? xabs : -xabs) > - ((yabs == y.data_) ? yabs : -yabs)); - } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x >= \a y - /// \retval false else - static bool isgreaterequal(half x, half y) { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - return xabs <= 0x7C00 && yabs <= 0x7C00 && - (((xabs == x.data_) ? xabs : -xabs) >= - ((yabs == y.data_) ? yabs : -yabs)); - } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x < \a y - /// \retval false else - static bool isless(half x, half y) { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - return xabs <= 0x7C00 && yabs <= 0x7C00 && - (((xabs == x.data_) ? xabs : -xabs) < - ((yabs == y.data_) ? yabs : -yabs)); - } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x <= \a y - /// \retval false else - static bool islessequal(half x, half y) { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - return xabs <= 0x7C00 && yabs <= 0x7C00 && - (((xabs == x.data_) ? xabs : -xabs) <= - ((yabs == y.data_) ? yabs : -yabs)); - } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if either \a x > \a y nor \a x < \a y - /// \retval false else - static bool islessgreater(half x, half y) { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - if (xabs > 0x7C00 || yabs > 0x7C00) return false; - int a = (xabs == x.data_) ? xabs : -xabs, - b = (yabs == y.data_) ? yabs : -yabs; - return a < b || a > b; - } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if operand unordered - /// \retval false else - static bool isunordered(half x, half y) { return isnan(x) || isnan(y); } - - private: - static double erf(double arg) { - if (builtin_isinf(arg)) return (arg < 0.0) ? -1.0 : 1.0; - double x2 = arg * arg, ax2 = 0.147 * x2, - value = std::sqrt( - 1.0 - std::exp(-x2 * (1.2732395447351626861510701069801 + ax2) / - (1.0 + ax2))); - return builtin_signbit(arg) ? -value : value; - } - - static double lgamma(double arg) { - double v = 1.0; - for (; arg < 8.0; ++arg) v *= arg; - double w = 1.0 / (arg * arg); - return (((((((-0.02955065359477124183006535947712 * w + - 0.00641025641025641025641025641026) * - w + - -0.00191752691752691752691752691753) * - w + - 8.4175084175084175084175084175084e-4) * - w + - -5.952380952380952380952380952381e-4) * - w + - 7.9365079365079365079365079365079e-4) * - w + - -0.00277777777777777777777777777778) * - w + - 0.08333333333333333333333333333333) / - arg + - 0.91893853320467274178032973640562 - std::log(v) - arg + - (arg - 0.5) * std::log(arg); - } -}; - -/// Wrapper for unary half-precision functions needing specialization for -/// individual argument types. -/// \tparam T argument type -template -struct unary_specialized { - /// Negation implementation. - /// \param arg value to negate - /// \return negated value - static HALF_CONSTEXPR half negate(half arg) { - return half(binary, arg.data_ ^ 0x8000); - } - - /// Absolute value implementation. - /// \param arg function argument - /// \return absolute value - static half fabs(half arg) { return half(binary, arg.data_ & 0x7FFF); } -}; -template <> -struct unary_specialized { - static HALF_CONSTEXPR expr negate(float arg) { return expr(-arg); } - static expr fabs(float arg) { return expr(std::fabs(arg)); } -}; - -/// Wrapper for binary half-precision functions needing specialization for -/// individual argument types. -/// \tparam T first argument type -/// \tparam U first argument type -template -struct binary_specialized { - /// Minimum implementation. - /// \param x first operand - /// \param y second operand - /// \return minimum value - static expr fmin(float x, float y) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::fmin(x, y)); -#else - if (builtin_isnan(x)) return expr(y); - if (builtin_isnan(y)) return expr(x); - return expr(std::min(x, y)); -#endif - } - - /// Maximum implementation. - /// \param x first operand - /// \param y second operand - /// \return maximum value - static expr fmax(float x, float y) { -#if HALF_ENABLE_CPP11_CMATH - return expr(std::fmax(x, y)); -#else - if (builtin_isnan(x)) return expr(y); - if (builtin_isnan(y)) return expr(x); - return expr(std::max(x, y)); -#endif - } -}; -template <> -struct binary_specialized { - static half fmin(half x, half y) { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - if (xabs > 0x7C00) return y; - if (yabs > 0x7C00) return x; - return (((xabs == x.data_) ? xabs : -xabs) > - ((yabs == y.data_) ? yabs : -yabs)) - ? y - : x; - } - static half fmax(half x, half y) { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - if (xabs > 0x7C00) return y; - if (yabs > 0x7C00) return x; - return (((xabs == x.data_) ? xabs : -xabs) < - ((yabs == y.data_) ? yabs : -yabs)) - ? y - : x; - } -}; - -/// Helper class for half casts. -/// This class template has to be specialized for all valid cast argument to -/// define an appropriate static `cast` member -/// function and a corresponding `type` member denoting its return type. -/// \tparam T destination type -/// \tparam U source type -/// \tparam R rounding mode to use -template -struct half_caster {}; -template -struct half_caster { -#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_arithmetic::value, - "half_cast from non-arithmetic type unsupported"); -#endif + /// Gamma implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr tgamma(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::tgamma(arg)); + #else + if(arg == 0.0f) + return builtin_signbit(arg) ? expr(-std::numeric_limits::infinity()) : expr(std::numeric_limits::infinity()); + if(arg < 0.0f) + { + float i, f = std::modf(-arg, &i); + if(f == 0.0f) + return expr(std::numeric_limits::quiet_NaN()); + double value = 3.1415926535897932384626433832795 / (std::sin(3.1415926535897932384626433832795*f)*std::exp(lgamma(1.0-arg))); + return expr(static_cast((std::fmod(i, 2.0f)==0.0f) ? -value : value)); + } + if(builtin_isinf(arg)) + return expr(arg); + return expr(static_cast(std::exp(lgamma(static_cast(arg))))); + #endif + } - static half cast(U arg) { return cast_impl(arg, is_float()); }; - - private: - static half cast_impl(U arg, true_type) { - return half(binary, float2half(arg)); - } - static half cast_impl(U arg, false_type) { - return half(binary, int2half(arg)); - } -}; -template -struct half_caster { -#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_arithmetic::value, - "half_cast to non-arithmetic type unsupported"); -#endif + /// Floor implementation. + /// \param arg value to round + /// \return rounded value + static half floor(half arg) { return half(binary, round_half(arg.data_)); } + + /// Ceiling implementation. + /// \param arg value to round + /// \return rounded value + static half ceil(half arg) { return half(binary, round_half(arg.data_)); } + + /// Truncation implementation. + /// \param arg value to round + /// \return rounded value + static half trunc(half arg) { return half(binary, round_half(arg.data_)); } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static half round(half arg) { return half(binary, round_half_up(arg.data_)); } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long lround(half arg) { return detail::half2int_up(arg.data_); } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static half rint(half arg) { return half(binary, round_half(arg.data_)); } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long lrint(half arg) { return detail::half2int(arg.data_); } + + #if HALF_ENABLE_CPP11_LONG_LONG + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long long llround(half arg) { return detail::half2int_up(arg.data_); } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long long llrint(half arg) { return detail::half2int(arg.data_); } + #endif + + /// Decompression implementation. + /// \param arg number to decompress + /// \param exp address to store exponent at + /// \return normalized significant + static half frexp(half arg, int *exp) + { + int m = arg.data_ & 0x7FFF, e = -14; + if(m >= 0x7C00 || !m) + return *exp = 0, arg; + for(; m<0x400; m<<=1,--e) ; + return *exp = e+(m>>10), half(binary, (arg.data_&0x8000)|0x3800|(m&0x3FF)); + } - static T cast(half arg) { return cast_impl(arg, is_float()); } - - private: - static T cast_impl(half arg, true_type) { return half2float(arg.data_); } - static T cast_impl(half arg, false_type) { return half2int(arg.data_); } -}; -template -struct half_caster { -#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_arithmetic::value, - "half_cast to non-arithmetic type unsupported"); -#endif + /// Decompression implementation. + /// \param arg number to decompress + /// \param iptr address to store integer part at + /// \return fractional part + static half modf(half arg, half *iptr) + { + unsigned int e = arg.data_ & 0x7FFF; + if(e >= 0x6400) + return *iptr = arg, half(binary, arg.data_&(0x8000U|-(e>0x7C00))); + if(e < 0x3C00) + return iptr->data_ = arg.data_ & 0x8000, arg; + e >>= 10; + unsigned int mask = (1<<(25-e)) - 1, m = arg.data_ & mask; + iptr->data_ = arg.data_ & ~mask; + if(!m) + return half(binary, arg.data_&0x8000); + for(; m<0x400; m<<=1,--e) ; + return half(binary, static_cast((arg.data_&0x8000)|(e<<10)|(m&0x3FF))); + } - static T cast(expr arg) { return cast_impl(arg, is_float()); } - - private: - static T cast_impl(float arg, true_type) { return static_cast(arg); } - static T cast_impl(half arg, false_type) { return half2int(arg.data_); } -}; -template -struct half_caster { - static half cast(half arg) { return arg; } -}; -template -struct half_caster : half_caster {}; - -/// \name Comparison operators -/// \{ - -/// Comparison for equality. -/// \param x first operand -/// \param y second operand -/// \retval true if operands equal -/// \retval false else -template -typename enable::type operator==(T x, U y) { - return functions::isequal(x, y); -} + /// Scaling implementation. + /// \param arg number to scale + /// \param exp power of two to scale by + /// \return scaled number + static half scalbln(half arg, long exp) + { + unsigned int m = arg.data_ & 0x7FFF; + if(m >= 0x7C00 || !m) + return arg; + for(; m<0x400; m<<=1,--exp) ; + exp += m >> 10; + uint16 value = arg.data_ & 0x8000; + if(exp > 30) + { + if(half::round_style == std::round_toward_zero) + value |= 0x7BFF; + else if(half::round_style == std::round_toward_infinity) + value |= 0x7C00 - (value>>15); + else if(half::round_style == std::round_toward_neg_infinity) + value |= 0x7BFF + (value>>15); + else + value |= 0x7C00; + } + else if(exp > 0) + value |= (exp<<10) | (m&0x3FF); + else if(exp > -11) + { + m = (m&0x3FF) | 0x400; + if(half::round_style == std::round_to_nearest) + { + m += 1 << -exp; + #if HALF_ROUND_TIES_TO_EVEN + m -= (m>>(1-exp)) & 1; + #endif + } + else if(half::round_style == std::round_toward_infinity) + m += ((value>>15)-1) & ((1<<(1-exp))-1U); + else if(half::round_style == std::round_toward_neg_infinity) + m += -(value>>15) & ((1<<(1-exp))-1U); + value |= m >> (1-exp); + } + else if(half::round_style == std::round_toward_infinity) + value -= (value>>15) - 1; + else if(half::round_style == std::round_toward_neg_infinity) + value += value >> 15; + return half(binary, value); + } -/// Comparison for inequality. -/// \param x first operand -/// \param y second operand -/// \retval true if operands not equal -/// \retval false else -template -typename enable::type operator!=(T x, U y) { - return functions::isnotequal(x, y); -} + /// Exponent implementation. + /// \param arg number to query + /// \return floating point exponent + static int ilogb(half arg) + { + int abs = arg.data_ & 0x7FFF; + if(!abs) + return FP_ILOGB0; + if(abs < 0x7C00) + { + int exp = (abs>>10) - 15; + if(abs < 0x400) + for(; abs<0x200; abs<<=1,--exp) ; + return exp; + } + if(abs > 0x7C00) + return FP_ILOGBNAN; + return INT_MAX; + } -/// Comparison for less than. -/// \param x first operand -/// \param y second operand -/// \retval true if \a x less than \a y -/// \retval false else -template -typename enable::type operator<(T x, U y) { - return functions::isless(x, y); -} + /// Exponent implementation. + /// \param arg number to query + /// \return floating point exponent + static half logb(half arg) + { + int abs = arg.data_ & 0x7FFF; + if(!abs) + return half(binary, 0xFC00); + if(abs < 0x7C00) + { + int exp = (abs>>10) - 15; + if(abs < 0x400) + for(; abs<0x200; abs<<=1,--exp) ; + uint16 bits = (exp<0) << 15; + if(exp) + { + unsigned int m = std::abs(exp) << 6, e = 18; + for(; m<0x400; m<<=1,--e) ; + bits |= (e<<10) + m; + } + return half(binary, bits); + } + if(abs > 0x7C00) + return arg; + return half(binary, 0x7C00); + } -/// Comparison for greater than. -/// \param x first operand -/// \param y second operand -/// \retval true if \a x greater than \a y -/// \retval false else -template -typename enable::type operator>(T x, U y) { - return functions::isgreater(x, y); -} + /// Enumeration implementation. + /// \param from number to increase/decrease + /// \param to direction to enumerate into + /// \return next representable number + static half nextafter(half from, half to) + { + uint16 fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF; + if(fabs > 0x7C00) + return from; + if(tabs > 0x7C00 || from.data_ == to.data_ || !(fabs|tabs)) + return to; + if(!fabs) + return half(binary, (to.data_&0x8000)+1); + bool lt = ((fabs==from.data_) ? static_cast(fabs) : -static_cast(fabs)) < + ((tabs==to.data_) ? static_cast(tabs) : -static_cast(tabs)); + return half(binary, from.data_+(((from.data_>>15)^static_cast(lt))<<1)-1); + } -/// Comparison for less equal. -/// \param x first operand -/// \param y second operand -/// \retval true if \a x less equal \a y -/// \retval false else -template -typename enable::type operator<=(T x, U y) { - return functions::islessequal(x, y); -} + /// Enumeration implementation. + /// \param from number to increase/decrease + /// \param to direction to enumerate into + /// \return next representable number + static half nexttoward(half from, long double to) + { + if(isnan(from)) + return from; + long double lfrom = static_cast(from); + if(builtin_isnan(to) || lfrom == to) + return half(static_cast(to)); + if(!(from.data_&0x7FFF)) + return half(binary, (static_cast(builtin_signbit(to))<<15)+1); + return half(binary, from.data_+(((from.data_>>15)^static_cast(lfrom -typename enable::type operator>=(T x, U y) { - return functions::isgreaterequal(x, y); -} + /// Sign implementation + /// \param x first operand + /// \param y second operand + /// \return composed value + static half copysign(half x, half y) { return half(binary, x.data_^((x.data_^y.data_)&0x8000)); } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if infinite number + /// \retval false else + static int fpclassify(half arg) + { + unsigned int abs = arg.data_ & 0x7FFF; + return abs ? ((abs>0x3FF) ? ((abs>=0x7C00) ? ((abs>0x7C00) ? FP_NAN : FP_INFINITE) : FP_NORMAL) :FP_SUBNORMAL) : FP_ZERO; + } -/// \} -/// \name Arithmetic operators -/// \{ - -/// Add halfs. -/// \param x left operand -/// \param y right operand -/// \return sum of half expressions -template -typename enable::type operator+(T x, U y) { - return functions::plus(x, y); -} + /// Classification implementation. + /// \param arg value to classify + /// \retval true if finite number + /// \retval false else + static bool isfinite(half arg) { return (arg.data_&0x7C00) != 0x7C00; } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if infinite number + /// \retval false else + static bool isinf(half arg) { return (arg.data_&0x7FFF) == 0x7C00; } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if not a number + /// \retval false else + static bool isnan(half arg) { return (arg.data_&0x7FFF) > 0x7C00; } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if normal number + /// \retval false else + static bool isnormal(half arg) { return ((arg.data_&0x7C00)!=0) & ((arg.data_&0x7C00)!=0x7C00); } + + /// Sign bit implementation. + /// \param arg value to check + /// \retval true if signed + /// \retval false if unsigned + static bool signbit(half arg) { return (arg.data_&0x8000) != 0; } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if operands equal + /// \retval false else + static bool isequal(half x, half y) { return (x.data_==y.data_ || !((x.data_|y.data_)&0x7FFF)) && !isnan(x); } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if operands not equal + /// \retval false else + static bool isnotequal(half x, half y) { return (x.data_!=y.data_ && ((x.data_|y.data_)&0x7FFF)) || isnan(x); } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x > \a y + /// \retval false else + static bool isgreater(half x, half y) + { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs)); + } -/// Subtract halfs. -/// \param x left operand -/// \param y right operand -/// \return difference of half expressions -template -typename enable::type operator-(T x, U y) { - return functions::minus(x, y); -} + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x >= \a y + /// \retval false else + static bool isgreaterequal(half x, half y) + { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) >= ((yabs==y.data_) ? yabs : -yabs)); + } -/// Multiply halfs. -/// \param x left operand -/// \param y right operand -/// \return product of half expressions -template -typename enable::type operator*(T x, U y) { - return functions::multiplies(x, y); -} + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x < \a y + /// \retval false else + static bool isless(half x, half y) + { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs)); + } -/// Divide halfs. -/// \param x left operand -/// \param y right operand -/// \return quotient of half expressions -template -typename enable::type operator/(T x, U y) { - return functions::divides(x, y); -} + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x <= \a y + /// \retval false else + static bool islessequal(half x, half y) + { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) <= ((yabs==y.data_) ? yabs : -yabs)); + } -/// Identity. -/// \param arg operand -/// \return uncahnged operand -template -HALF_CONSTEXPR typename enable::type operator+(T arg) { - return arg; -} + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if either \a x > \a y nor \a x < \a y + /// \retval false else + static bool islessgreater(half x, half y) + { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + if(xabs > 0x7C00 || yabs > 0x7C00) + return false; + int a = (xabs==x.data_) ? xabs : -xabs, b = (yabs==y.data_) ? yabs : -yabs; + return a < b || a > b; + } -/// Negation. -/// \param arg operand -/// \return negated operand -template -HALF_CONSTEXPR typename enable::type operator-(T arg) { - return unary_specialized::negate(arg); -} + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if operand unordered + /// \retval false else + static bool isunordered(half x, half y) { return isnan(x) || isnan(y); } -/// \} -/// \name Input and output -/// \{ - -/// Output operator. -/// \param out output stream to write into -/// \param arg half expression to write -/// \return reference to output stream -template -typename enable &, T>::type operator<<( - std::basic_ostream &out, T arg) { - return functions::write(out, arg); -} + private: + static double erf(double arg) + { + if(builtin_isinf(arg)) + return (arg<0.0) ? -1.0 : 1.0; + double x2 = arg * arg, ax2 = 0.147 * x2, value = std::sqrt(1.0-std::exp(-x2*(1.2732395447351626861510701069801+ax2)/(1.0+ax2))); + return builtin_signbit(arg) ? -value : value; + } -/// Input operator. -/// \param in input stream to read from -/// \param arg half to read into -/// \return reference to input stream -template -std::basic_istream &operator>>( - std::basic_istream &in, half &arg) { - return functions::read(in, arg); -} + static double lgamma(double arg) + { + double v = 1.0; + for(; arg<8.0; ++arg) v *= arg; + double w = 1.0 / (arg*arg); + return (((((((-0.02955065359477124183006535947712*w+0.00641025641025641025641025641026)*w+ + -0.00191752691752691752691752691753)*w+8.4175084175084175084175084175084e-4)*w+ + -5.952380952380952380952380952381e-4)*w+7.9365079365079365079365079365079e-4)*w+ + -0.00277777777777777777777777777778)*w+0.08333333333333333333333333333333)/arg + + 0.91893853320467274178032973640562 - std::log(v) - arg + (arg-0.5) * std::log(arg); + } + }; + + /// Wrapper for unary half-precision functions needing specialization for individual argument types. + /// \tparam T argument type + template struct unary_specialized + { + /// Negation implementation. + /// \param arg value to negate + /// \return negated value + static HALF_CONSTEXPR half negate(half arg) { return half(binary, arg.data_^0x8000); } + + /// Absolute value implementation. + /// \param arg function argument + /// \return absolute value + static half fabs(half arg) { return half(binary, arg.data_&0x7FFF); } + }; + template<> struct unary_specialized + { + static HALF_CONSTEXPR expr negate(float arg) { return expr(-arg); } + static expr fabs(float arg) { return expr(std::fabs(arg)); } + }; + + /// Wrapper for binary half-precision functions needing specialization for individual argument types. + /// \tparam T first argument type + /// \tparam U first argument type + template struct binary_specialized + { + /// Minimum implementation. + /// \param x first operand + /// \param y second operand + /// \return minimum value + static expr fmin(float x, float y) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::fmin(x, y)); + #else + if(builtin_isnan(x)) + return expr(y); + if(builtin_isnan(y)) + return expr(x); + return expr(std::min(x, y)); + #endif + } -/// \} -/// \name Basic mathematical operations -/// \{ - -/// Absolute value. -/// \param arg operand -/// \return absolute value of \a arg -// template typename enable::type abs(T arg) { -//return unary_specialized::fabs(arg); } -inline half abs(half arg) { return unary_specialized::fabs(arg); } -inline expr abs(expr arg) { return unary_specialized::fabs(arg); } - -/// Absolute value. -/// \param arg operand -/// \return absolute value of \a arg -// template typename enable::type fabs(T arg) { -//return unary_specialized::fabs(arg); } -inline half fabs(half arg) { return unary_specialized::fabs(arg); } -inline expr fabs(expr arg) { return unary_specialized::fabs(arg); } - -/// Remainder of division. -/// \param x first operand -/// \param y second operand -/// \return remainder of floating point division. -// template typename enable::type -//fmod(T x, U y) { return functions::fmod(x, y); } -inline expr fmod(half x, half y) { return functions::fmod(x, y); } -inline expr fmod(half x, expr y) { return functions::fmod(x, y); } -inline expr fmod(expr x, half y) { return functions::fmod(x, y); } -inline expr fmod(expr x, expr y) { return functions::fmod(x, y); } - -/// Remainder of division. -/// \param x first operand -/// \param y second operand -/// \return remainder of floating point division. -// template typename enable::type -//remainder(T x, U y) { return functions::remainder(x, y); } -inline expr remainder(half x, half y) { return functions::remainder(x, y); } -inline expr remainder(half x, expr y) { return functions::remainder(x, y); } -inline expr remainder(expr x, half y) { return functions::remainder(x, y); } -inline expr remainder(expr x, expr y) { return functions::remainder(x, y); } - -/// Remainder of division. -/// \param x first operand -/// \param y second operand -/// \param quo address to store some bits of quotient at -/// \return remainder of floating point division. -// template typename enable::type -//remquo(T x, U y, int *quo) { return functions::remquo(x, y, quo); } -inline expr remquo(half x, half y, int *quo) { - return functions::remquo(x, y, quo); -} -inline expr remquo(half x, expr y, int *quo) { - return functions::remquo(x, y, quo); -} -inline expr remquo(expr x, half y, int *quo) { - return functions::remquo(x, y, quo); -} -inline expr remquo(expr x, expr y, int *quo) { - return functions::remquo(x, y, quo); + /// Maximum implementation. + /// \param x first operand + /// \param y second operand + /// \return maximum value + static expr fmax(float x, float y) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::fmax(x, y)); + #else + if(builtin_isnan(x)) + return expr(y); + if(builtin_isnan(y)) + return expr(x); + return expr(std::max(x, y)); + #endif + } + }; + template<> struct binary_specialized + { + static half fmin(half x, half y) + { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + if(xabs > 0x7C00) + return y; + if(yabs > 0x7C00) + return x; + return (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs)) ? y : x; + } + static half fmax(half x, half y) + { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + if(xabs > 0x7C00) + return y; + if(yabs > 0x7C00) + return x; + return (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs)) ? y : x; + } + }; + + /// Helper class for half casts. + /// This class template has to be specialized for all valid cast argument to define an appropriate static `cast` member + /// function and a corresponding `type` member denoting its return type. + /// \tparam T destination type + /// \tparam U source type + /// \tparam R rounding mode to use + template struct half_caster {}; + template struct half_caster + { + #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, "half_cast from non-arithmetic type unsupported"); + #endif + + static half cast(U arg) { return cast_impl(arg, is_float()); }; + + private: + static half cast_impl(U arg, true_type) { return half(binary, float2half(arg)); } + static half cast_impl(U arg, false_type) { return half(binary, int2half(arg)); } + }; + template struct half_caster + { + #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, "half_cast to non-arithmetic type unsupported"); + #endif + + static T cast(half arg) { return cast_impl(arg, is_float()); } + + private: + static T cast_impl(half arg, true_type) { return half2float(arg.data_); } + static T cast_impl(half arg, false_type) { return half2int(arg.data_); } + }; + template struct half_caster + { + #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, "half_cast to non-arithmetic type unsupported"); + #endif + + static T cast(expr arg) { return cast_impl(arg, is_float()); } + + private: + static T cast_impl(float arg, true_type) { return static_cast(arg); } + static T cast_impl(half arg, false_type) { return half2int(arg.data_); } + }; + template struct half_caster + { + static half cast(half arg) { return arg; } + }; + template struct half_caster : half_caster {}; + + /// \name Comparison operators + /// \{ + + /// Comparison for equality. + /// \param x first operand + /// \param y second operand + /// \retval true if operands equal + /// \retval false else + template typename enable::type operator==(T x, U y) { return functions::isequal(x, y); } + + /// Comparison for inequality. + /// \param x first operand + /// \param y second operand + /// \retval true if operands not equal + /// \retval false else + template typename enable::type operator!=(T x, U y) { return functions::isnotequal(x, y); } + + /// Comparison for less than. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less than \a y + /// \retval false else + template typename enable::type operator<(T x, U y) { return functions::isless(x, y); } + + /// Comparison for greater than. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater than \a y + /// \retval false else + template typename enable::type operator>(T x, U y) { return functions::isgreater(x, y); } + + /// Comparison for less equal. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less equal \a y + /// \retval false else + template typename enable::type operator<=(T x, U y) { return functions::islessequal(x, y); } + + /// Comparison for greater equal. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater equal \a y + /// \retval false else + template typename enable::type operator>=(T x, U y) { return functions::isgreaterequal(x, y); } + + /// \} + /// \name Arithmetic operators + /// \{ + + /// Add halfs. + /// \param x left operand + /// \param y right operand + /// \return sum of half expressions + template typename enable::type operator+(T x, U y) { return functions::plus(x, y); } + + /// Subtract halfs. + /// \param x left operand + /// \param y right operand + /// \return difference of half expressions + template typename enable::type operator-(T x, U y) { return functions::minus(x, y); } + + /// Multiply halfs. + /// \param x left operand + /// \param y right operand + /// \return product of half expressions + template typename enable::type operator*(T x, U y) { return functions::multiplies(x, y); } + + /// Divide halfs. + /// \param x left operand + /// \param y right operand + /// \return quotient of half expressions + template typename enable::type operator/(T x, U y) { return functions::divides(x, y); } + + /// Identity. + /// \param arg operand + /// \return uncahnged operand + template HALF_CONSTEXPR typename enable::type operator+(T arg) { return arg; } + + /// Negation. + /// \param arg operand + /// \return negated operand + template HALF_CONSTEXPR typename enable::type operator-(T arg) { return unary_specialized::negate(arg); } + + /// \} + /// \name Input and output + /// \{ + + /// Output operator. + /// \param out output stream to write into + /// \param arg half expression to write + /// \return reference to output stream + template typename enable&,T>::type + operator<<(std::basic_ostream &out, T arg) { return functions::write(out, arg); } + + /// Input operator. + /// \param in input stream to read from + /// \param arg half to read into + /// \return reference to input stream + template std::basic_istream& + operator>>(std::basic_istream &in, half &arg) { return functions::read(in, arg); } + + /// \} + /// \name Basic mathematical operations + /// \{ + + /// Absolute value. + /// \param arg operand + /// \return absolute value of \a arg +// template typename enable::type abs(T arg) { return unary_specialized::fabs(arg); } + inline half abs(half arg) { return unary_specialized::fabs(arg); } + inline expr abs(expr arg) { return unary_specialized::fabs(arg); } + + /// Absolute value. + /// \param arg operand + /// \return absolute value of \a arg +// template typename enable::type fabs(T arg) { return unary_specialized::fabs(arg); } + inline half fabs(half arg) { return unary_specialized::fabs(arg); } + inline expr fabs(expr arg) { return unary_specialized::fabs(arg); } + + /// Remainder of division. + /// \param x first operand + /// \param y second operand + /// \return remainder of floating point division. +// template typename enable::type fmod(T x, U y) { return functions::fmod(x, y); } + inline expr fmod(half x, half y) { return functions::fmod(x, y); } + inline expr fmod(half x, expr y) { return functions::fmod(x, y); } + inline expr fmod(expr x, half y) { return functions::fmod(x, y); } + inline expr fmod(expr x, expr y) { return functions::fmod(x, y); } + + /// Remainder of division. + /// \param x first operand + /// \param y second operand + /// \return remainder of floating point division. +// template typename enable::type remainder(T x, U y) { return functions::remainder(x, y); } + inline expr remainder(half x, half y) { return functions::remainder(x, y); } + inline expr remainder(half x, expr y) { return functions::remainder(x, y); } + inline expr remainder(expr x, half y) { return functions::remainder(x, y); } + inline expr remainder(expr x, expr y) { return functions::remainder(x, y); } + + /// Remainder of division. + /// \param x first operand + /// \param y second operand + /// \param quo address to store some bits of quotient at + /// \return remainder of floating point division. +// template typename enable::type remquo(T x, U y, int *quo) { return functions::remquo(x, y, quo); } + inline expr remquo(half x, half y, int *quo) { return functions::remquo(x, y, quo); } + inline expr remquo(half x, expr y, int *quo) { return functions::remquo(x, y, quo); } + inline expr remquo(expr x, half y, int *quo) { return functions::remquo(x, y, quo); } + inline expr remquo(expr x, expr y, int *quo) { return functions::remquo(x, y, quo); } + + /// Fused multiply add. + /// \param x first operand + /// \param y second operand + /// \param z third operand + /// \return ( \a x * \a y ) + \a z rounded as one operation. +// template typename enable::type fma(T x, U y, V z) { return functions::fma(x, y, z); } + inline expr fma(half x, half y, half z) { return functions::fma(x, y, z); } + inline expr fma(half x, half y, expr z) { return functions::fma(x, y, z); } + inline expr fma(half x, expr y, half z) { return functions::fma(x, y, z); } + inline expr fma(half x, expr y, expr z) { return functions::fma(x, y, z); } + inline expr fma(expr x, half y, half z) { return functions::fma(x, y, z); } + inline expr fma(expr x, half y, expr z) { return functions::fma(x, y, z); } + inline expr fma(expr x, expr y, half z) { return functions::fma(x, y, z); } + inline expr fma(expr x, expr y, expr z) { return functions::fma(x, y, z); } + + /// Maximum of half expressions. + /// \param x first operand + /// \param y second operand + /// \return maximum of operands +// template typename result::type fmax(T x, U y) { return binary_specialized::fmax(x, y); } + inline half fmax(half x, half y) { return binary_specialized::fmax(x, y); } + inline expr fmax(half x, expr y) { return binary_specialized::fmax(x, y); } + inline expr fmax(expr x, half y) { return binary_specialized::fmax(x, y); } + inline expr fmax(expr x, expr y) { return binary_specialized::fmax(x, y); } + + /// Minimum of half expressions. + /// \param x first operand + /// \param y second operand + /// \return minimum of operands +// template typename result::type fmin(T x, U y) { return binary_specialized::fmin(x, y); } + inline half fmin(half x, half y) { return binary_specialized::fmin(x, y); } + inline expr fmin(half x, expr y) { return binary_specialized::fmin(x, y); } + inline expr fmin(expr x, half y) { return binary_specialized::fmin(x, y); } + inline expr fmin(expr x, expr y) { return binary_specialized::fmin(x, y); } + + /// Positive difference. + /// \param x first operand + /// \param y second operand + /// \return \a x - \a y or 0 if difference negative +// template typename enable::type fdim(T x, U y) { return functions::fdim(x, y); } + inline expr fdim(half x, half y) { return functions::fdim(x, y); } + inline expr fdim(half x, expr y) { return functions::fdim(x, y); } + inline expr fdim(expr x, half y) { return functions::fdim(x, y); } + inline expr fdim(expr x, expr y) { return functions::fdim(x, y); } + + /// Get NaN value. + /// \return quiet NaN + inline half nanh(const char*) { return functions::nanh(); } + + /// \} + /// \name Exponential functions + /// \{ + + /// Exponential function. + /// \param arg function argument + /// \return e raised to \a arg +// template typename enable::type exp(T arg) { return functions::exp(arg); } + inline expr exp(half arg) { return functions::exp(arg); } + inline expr exp(expr arg) { return functions::exp(arg); } + + /// Exponential minus one. + /// \param arg function argument + /// \return e raised to \a arg subtracted by 1 +// template typename enable::type expm1(T arg) { return functions::expm1(arg); } + inline expr expm1(half arg) { return functions::expm1(arg); } + inline expr expm1(expr arg) { return functions::expm1(arg); } + + /// Binary exponential. + /// \param arg function argument + /// \return 2 raised to \a arg +// template typename enable::type exp2(T arg) { return functions::exp2(arg); } + inline expr exp2(half arg) { return functions::exp2(arg); } + inline expr exp2(expr arg) { return functions::exp2(arg); } + + /// Natural logorithm. + /// \param arg function argument + /// \return logarithm of \a arg to base e +// template typename enable::type log(T arg) { return functions::log(arg); } + inline expr log(half arg) { return functions::log(arg); } + inline expr log(expr arg) { return functions::log(arg); } + + /// Common logorithm. + /// \param arg function argument + /// \return logarithm of \a arg to base 10 +// template typename enable::type log10(T arg) { return functions::log10(arg); } + inline expr log10(half arg) { return functions::log10(arg); } + inline expr log10(expr arg) { return functions::log10(arg); } + + /// Natural logorithm. + /// \param arg function argument + /// \return logarithm of \a arg plus 1 to base e +// template typename enable::type log1p(T arg) { return functions::log1p(arg); } + inline expr log1p(half arg) { return functions::log1p(arg); } + inline expr log1p(expr arg) { return functions::log1p(arg); } + + /// Binary logorithm. + /// \param arg function argument + /// \return logarithm of \a arg to base 2 +// template typename enable::type log2(T arg) { return functions::log2(arg); } + inline expr log2(half arg) { return functions::log2(arg); } + inline expr log2(expr arg) { return functions::log2(arg); } + + /// \} + /// \name Power functions + /// \{ + + /// Square root. + /// \param arg function argument + /// \return square root of \a arg +// template typename enable::type sqrt(T arg) { return functions::sqrt(arg); } + inline expr sqrt(half arg) { return functions::sqrt(arg); } + inline expr sqrt(expr arg) { return functions::sqrt(arg); } + + /// Cubic root. + /// \param arg function argument + /// \return cubic root of \a arg +// template typename enable::type cbrt(T arg) { return functions::cbrt(arg); } + inline expr cbrt(half arg) { return functions::cbrt(arg); } + inline expr cbrt(expr arg) { return functions::cbrt(arg); } + + /// Hypotenuse function. + /// \param x first argument + /// \param y second argument + /// \return square root of sum of squares without internal over- or underflows +// template typename enable::type hypot(T x, U y) { return functions::hypot(x, y); } + inline expr hypot(half x, half y) { return functions::hypot(x, y); } + inline expr hypot(half x, expr y) { return functions::hypot(x, y); } + inline expr hypot(expr x, half y) { return functions::hypot(x, y); } + inline expr hypot(expr x, expr y) { return functions::hypot(x, y); } + + /// Power function. + /// \param base first argument + /// \param exp second argument + /// \return \a base raised to \a exp +// template typename enable::type pow(T base, U exp) { return functions::pow(base, exp); } + inline expr pow(half base, half exp) { return functions::pow(base, exp); } + inline expr pow(half base, expr exp) { return functions::pow(base, exp); } + inline expr pow(expr base, half exp) { return functions::pow(base, exp); } + inline expr pow(expr base, expr exp) { return functions::pow(base, exp); } + + /// \} + /// \name Trigonometric functions + /// \{ + + /// Sine function. + /// \param arg function argument + /// \return sine value of \a arg +// template typename enable::type sin(T arg) { return functions::sin(arg); } + inline expr sin(half arg) { return functions::sin(arg); } + inline expr sin(expr arg) { return functions::sin(arg); } + + /// Cosine function. + /// \param arg function argument + /// \return cosine value of \a arg +// template typename enable::type cos(T arg) { return functions::cos(arg); } + inline expr cos(half arg) { return functions::cos(arg); } + inline expr cos(expr arg) { return functions::cos(arg); } + + /// Tangent function. + /// \param arg function argument + /// \return tangent value of \a arg +// template typename enable::type tan(T arg) { return functions::tan(arg); } + inline expr tan(half arg) { return functions::tan(arg); } + inline expr tan(expr arg) { return functions::tan(arg); } + + /// Arc sine. + /// \param arg function argument + /// \return arc sine value of \a arg +// template typename enable::type asin(T arg) { return functions::asin(arg); } + inline expr asin(half arg) { return functions::asin(arg); } + inline expr asin(expr arg) { return functions::asin(arg); } + + /// Arc cosine function. + /// \param arg function argument + /// \return arc cosine value of \a arg +// template typename enable::type acos(T arg) { return functions::acos(arg); } + inline expr acos(half arg) { return functions::acos(arg); } + inline expr acos(expr arg) { return functions::acos(arg); } + + /// Arc tangent function. + /// \param arg function argument + /// \return arc tangent value of \a arg +// template typename enable::type atan(T arg) { return functions::atan(arg); } + inline expr atan(half arg) { return functions::atan(arg); } + inline expr atan(expr arg) { return functions::atan(arg); } + + /// Arc tangent function. + /// \param x first argument + /// \param y second argument + /// \return arc tangent value +// template typename enable::type atan2(T x, U y) { return functions::atan2(x, y); } + inline expr atan2(half x, half y) { return functions::atan2(x, y); } + inline expr atan2(half x, expr y) { return functions::atan2(x, y); } + inline expr atan2(expr x, half y) { return functions::atan2(x, y); } + inline expr atan2(expr x, expr y) { return functions::atan2(x, y); } + + /// \} + /// \name Hyperbolic functions + /// \{ + + /// Hyperbolic sine. + /// \param arg function argument + /// \return hyperbolic sine value of \a arg +// template typename enable::type sinh(T arg) { return functions::sinh(arg); } + inline expr sinh(half arg) { return functions::sinh(arg); } + inline expr sinh(expr arg) { return functions::sinh(arg); } + + /// Hyperbolic cosine. + /// \param arg function argument + /// \return hyperbolic cosine value of \a arg +// template typename enable::type cosh(T arg) { return functions::cosh(arg); } + inline expr cosh(half arg) { return functions::cosh(arg); } + inline expr cosh(expr arg) { return functions::cosh(arg); } + + /// Hyperbolic tangent. + /// \param arg function argument + /// \return hyperbolic tangent value of \a arg +// template typename enable::type tanh(T arg) { return functions::tanh(arg); } + inline expr tanh(half arg) { return functions::tanh(arg); } + inline expr tanh(expr arg) { return functions::tanh(arg); } + + /// Hyperbolic area sine. + /// \param arg function argument + /// \return area sine value of \a arg +// template typename enable::type asinh(T arg) { return functions::asinh(arg); } + inline expr asinh(half arg) { return functions::asinh(arg); } + inline expr asinh(expr arg) { return functions::asinh(arg); } + + /// Hyperbolic area cosine. + /// \param arg function argument + /// \return area cosine value of \a arg +// template typename enable::type acosh(T arg) { return functions::acosh(arg); } + inline expr acosh(half arg) { return functions::acosh(arg); } + inline expr acosh(expr arg) { return functions::acosh(arg); } + + /// Hyperbolic area tangent. + /// \param arg function argument + /// \return area tangent value of \a arg +// template typename enable::type atanh(T arg) { return functions::atanh(arg); } + inline expr atanh(half arg) { return functions::atanh(arg); } + inline expr atanh(expr arg) { return functions::atanh(arg); } + + /// \} + /// \name Error and gamma functions + /// \{ + + /// Error function. + /// \param arg function argument + /// \return error function value of \a arg +// template typename enable::type erf(T arg) { return functions::erf(arg); } + inline expr erf(half arg) { return functions::erf(arg); } + inline expr erf(expr arg) { return functions::erf(arg); } + + /// Complementary error function. + /// \param arg function argument + /// \return 1 minus error function value of \a arg +// template typename enable::type erfc(T arg) { return functions::erfc(arg); } + inline expr erfc(half arg) { return functions::erfc(arg); } + inline expr erfc(expr arg) { return functions::erfc(arg); } + + /// Natural logarithm of gamma function. + /// \param arg function argument + /// \return natural logarith of gamma function for \a arg +// template typename enable::type lgamma(T arg) { return functions::lgamma(arg); } + inline expr lgamma(half arg) { return functions::lgamma(arg); } + inline expr lgamma(expr arg) { return functions::lgamma(arg); } + + /// Gamma function. + /// \param arg function argument + /// \return gamma function value of \a arg +// template typename enable::type tgamma(T arg) { return functions::tgamma(arg); } + inline expr tgamma(half arg) { return functions::tgamma(arg); } + inline expr tgamma(expr arg) { return functions::tgamma(arg); } + + /// \} + /// \name Rounding + /// \{ + + /// Nearest integer not less than half value. + /// \param arg half to round + /// \return nearest integer not less than \a arg +// template typename enable::type ceil(T arg) { return functions::ceil(arg); } + inline half ceil(half arg) { return functions::ceil(arg); } + inline half ceil(expr arg) { return functions::ceil(arg); } + + /// Nearest integer not greater than half value. + /// \param arg half to round + /// \return nearest integer not greater than \a arg +// template typename enable::type floor(T arg) { return functions::floor(arg); } + inline half floor(half arg) { return functions::floor(arg); } + inline half floor(expr arg) { return functions::floor(arg); } + + /// Nearest integer not greater in magnitude than half value. + /// \param arg half to round + /// \return nearest integer not greater in magnitude than \a arg +// template typename enable::type trunc(T arg) { return functions::trunc(arg); } + inline half trunc(half arg) { return functions::trunc(arg); } + inline half trunc(expr arg) { return functions::trunc(arg); } + + /// Nearest integer. + /// \param arg half to round + /// \return nearest integer, rounded away from zero in half-way cases +// template typename enable::type round(T arg) { return functions::round(arg); } + inline half round(half arg) { return functions::round(arg); } + inline half round(expr arg) { return functions::round(arg); } + + /// Nearest integer. + /// \param arg half to round + /// \return nearest integer, rounded away from zero in half-way cases +// template typename enable::type lround(T arg) { return functions::lround(arg); } + inline long lround(half arg) { return functions::lround(arg); } + inline long lround(expr arg) { return functions::lround(arg); } + + /// Nearest integer using half's internal rounding mode. + /// \param arg half expression to round + /// \return nearest integer using default rounding mode +// template typename enable::type nearbyint(T arg) { return functions::nearbyint(arg); } + inline half nearbyint(half arg) { return functions::rint(arg); } + inline half nearbyint(expr arg) { return functions::rint(arg); } + + /// Nearest integer using half's internal rounding mode. + /// \param arg half expression to round + /// \return nearest integer using default rounding mode +// template typename enable::type rint(T arg) { return functions::rint(arg); } + inline half rint(half arg) { return functions::rint(arg); } + inline half rint(expr arg) { return functions::rint(arg); } + + /// Nearest integer using half's internal rounding mode. + /// \param arg half expression to round + /// \return nearest integer using default rounding mode +// template typename enable::type lrint(T arg) { return functions::lrint(arg); } + inline long lrint(half arg) { return functions::lrint(arg); } + inline long lrint(expr arg) { return functions::lrint(arg); } + #if HALF_ENABLE_CPP11_LONG_LONG + /// Nearest integer. + /// \param arg half to round + /// \return nearest integer, rounded away from zero in half-way cases +// template typename enable::type llround(T arg) { return functions::llround(arg); } + inline long long llround(half arg) { return functions::llround(arg); } + inline long long llround(expr arg) { return functions::llround(arg); } + + /// Nearest integer using half's internal rounding mode. + /// \param arg half expression to round + /// \return nearest integer using default rounding mode +// template typename enable::type llrint(T arg) { return functions::llrint(arg); } + inline long long llrint(half arg) { return functions::llrint(arg); } + inline long long llrint(expr arg) { return functions::llrint(arg); } + #endif + + /// \} + /// \name Floating point manipulation + /// \{ + + /// Decompress floating point number. + /// \param arg number to decompress + /// \param exp address to store exponent at + /// \return significant in range [0.5, 1) +// template typename enable::type frexp(T arg, int *exp) { return functions::frexp(arg, exp); } + inline half frexp(half arg, int *exp) { return functions::frexp(arg, exp); } + inline half frexp(expr arg, int *exp) { return functions::frexp(arg, exp); } + + /// Multiply by power of two. + /// \param arg number to modify + /// \param exp power of two to multiply with + /// \return \a arg multplied by 2 raised to \a exp +// template typename enable::type ldexp(T arg, int exp) { return functions::scalbln(arg, exp); } + inline half ldexp(half arg, int exp) { return functions::scalbln(arg, exp); } + inline half ldexp(expr arg, int exp) { return functions::scalbln(arg, exp); } + + /// Extract integer and fractional parts. + /// \param arg number to decompress + /// \param iptr address to store integer part at + /// \return fractional part +// template typename enable::type modf(T arg, half *iptr) { return functions::modf(arg, iptr); } + inline half modf(half arg, half *iptr) { return functions::modf(arg, iptr); } + inline half modf(expr arg, half *iptr) { return functions::modf(arg, iptr); } + + /// Multiply by power of two. + /// \param arg number to modify + /// \param exp power of two to multiply with + /// \return \a arg multplied by 2 raised to \a exp +// template typename enable::type scalbn(T arg, int exp) { return functions::scalbln(arg, exp); } + inline half scalbn(half arg, int exp) { return functions::scalbln(arg, exp); } + inline half scalbn(expr arg, int exp) { return functions::scalbln(arg, exp); } + + /// Multiply by power of two. + /// \param arg number to modify + /// \param exp power of two to multiply with + /// \return \a arg multplied by 2 raised to \a exp +// template typename enable::type scalbln(T arg, long exp) { return functions::scalbln(arg, exp); } + inline half scalbln(half arg, long exp) { return functions::scalbln(arg, exp); } + inline half scalbln(expr arg, long exp) { return functions::scalbln(arg, exp); } + + /// Extract exponent. + /// \param arg number to query + /// \return floating point exponent + /// \retval FP_ILOGB0 for zero + /// \retval FP_ILOGBNAN for NaN + /// \retval MAX_INT for infinity +// template typename enable::type ilogb(T arg) { return functions::ilogb(arg); } + inline int ilogb(half arg) { return functions::ilogb(arg); } + inline int ilogb(expr arg) { return functions::ilogb(arg); } + + /// Extract exponent. + /// \param arg number to query + /// \return floating point exponent +// template typename enable::type logb(T arg) { return functions::logb(arg); } + inline half logb(half arg) { return functions::logb(arg); } + inline half logb(expr arg) { return functions::logb(arg); } + + /// Next representable value. + /// \param from value to compute next representable value for + /// \param to direction towards which to compute next value + /// \return next representable value after \a from in direction towards \a to +// template typename enable::type nextafter(T from, U to) { return functions::nextafter(from, to); } + inline half nextafter(half from, half to) { return functions::nextafter(from, to); } + inline half nextafter(half from, expr to) { return functions::nextafter(from, to); } + inline half nextafter(expr from, half to) { return functions::nextafter(from, to); } + inline half nextafter(expr from, expr to) { return functions::nextafter(from, to); } + + /// Next representable value. + /// \param from value to compute next representable value for + /// \param to direction towards which to compute next value + /// \return next representable value after \a from in direction towards \a to +// template typename enable::type nexttoward(T from, long double to) { return functions::nexttoward(from, to); } + inline half nexttoward(half from, long double to) { return functions::nexttoward(from, to); } + inline half nexttoward(expr from, long double to) { return functions::nexttoward(from, to); } + + /// Take sign. + /// \param x value to change sign for + /// \param y value to take sign from + /// \return value equal to \a x in magnitude and to \a y in sign +// template typename enable::type copysign(T x, U y) { return functions::copysign(x, y); } + inline half copysign(half x, half y) { return functions::copysign(x, y); } + inline half copysign(half x, expr y) { return functions::copysign(x, y); } + inline half copysign(expr x, half y) { return functions::copysign(x, y); } + inline half copysign(expr x, expr y) { return functions::copysign(x, y); } + + /// \} + /// \name Floating point classification + /// \{ + + + /// Classify floating point value. + /// \param arg number to classify + /// \retval FP_ZERO for positive and negative zero + /// \retval FP_SUBNORMAL for subnormal numbers + /// \retval FP_INFINITY for positive and negative infinity + /// \retval FP_NAN for NaNs + /// \retval FP_NORMAL for all other (normal) values +// template typename enable::type fpclassify(T arg) { return functions::fpclassify(arg); } + inline int fpclassify(half arg) { return functions::fpclassify(arg); } + inline int fpclassify(expr arg) { return functions::fpclassify(arg); } + + /// Check if finite number. + /// \param arg number to check + /// \retval true if neither infinity nor NaN + /// \retval false else +// template typename enable::type isfinite(T arg) { return functions::isfinite(arg); } + inline bool isfinite(half arg) { return functions::isfinite(arg); } + inline bool isfinite(expr arg) { return functions::isfinite(arg); } + + /// Check for infinity. + /// \param arg number to check + /// \retval true for positive or negative infinity + /// \retval false else +// template typename enable::type isinf(T arg) { return functions::isinf(arg); } + inline bool isinf(half arg) { return functions::isinf(arg); } + inline bool isinf(expr arg) { return functions::isinf(arg); } + + /// Check for NaN. + /// \param arg number to check + /// \retval true for NaNs + /// \retval false else +// template typename enable::type isnan(T arg) { return functions::isnan(arg); } + inline bool isnan(half arg) { return functions::isnan(arg); } + inline bool isnan(expr arg) { return functions::isnan(arg); } + + /// Check if normal number. + /// \param arg number to check + /// \retval true if normal number + /// \retval false if either subnormal, zero, infinity or NaN +// template typename enable::type isnormal(T arg) { return functions::isnormal(arg); } + inline bool isnormal(half arg) { return functions::isnormal(arg); } + inline bool isnormal(expr arg) { return functions::isnormal(arg); } + + /// Check sign. + /// \param arg number to check + /// \retval true for negative number + /// \retval false for positive number +// template typename enable::type signbit(T arg) { return functions::signbit(arg); } + inline bool signbit(half arg) { return functions::signbit(arg); } + inline bool signbit(expr arg) { return functions::signbit(arg); } + + /// \} + /// \name Comparison + /// \{ + + /// Comparison for greater than. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater than \a y + /// \retval false else +// template typename enable::type isgreater(T x, U y) { return functions::isgreater(x, y); } + inline bool isgreater(half x, half y) { return functions::isgreater(x, y); } + inline bool isgreater(half x, expr y) { return functions::isgreater(x, y); } + inline bool isgreater(expr x, half y) { return functions::isgreater(x, y); } + inline bool isgreater(expr x, expr y) { return functions::isgreater(x, y); } + + /// Comparison for greater equal. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater equal \a y + /// \retval false else +// template typename enable::type isgreaterequal(T x, U y) { return functions::isgreaterequal(x, y); } + inline bool isgreaterequal(half x, half y) { return functions::isgreaterequal(x, y); } + inline bool isgreaterequal(half x, expr y) { return functions::isgreaterequal(x, y); } + inline bool isgreaterequal(expr x, half y) { return functions::isgreaterequal(x, y); } + inline bool isgreaterequal(expr x, expr y) { return functions::isgreaterequal(x, y); } + + /// Comparison for less than. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less than \a y + /// \retval false else +// template typename enable::type isless(T x, U y) { return functions::isless(x, y); } + inline bool isless(half x, half y) { return functions::isless(x, y); } + inline bool isless(half x, expr y) { return functions::isless(x, y); } + inline bool isless(expr x, half y) { return functions::isless(x, y); } + inline bool isless(expr x, expr y) { return functions::isless(x, y); } + + /// Comparison for less equal. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less equal \a y + /// \retval false else +// template typename enable::type islessequal(T x, U y) { return functions::islessequal(x, y); } + inline bool islessequal(half x, half y) { return functions::islessequal(x, y); } + inline bool islessequal(half x, expr y) { return functions::islessequal(x, y); } + inline bool islessequal(expr x, half y) { return functions::islessequal(x, y); } + inline bool islessequal(expr x, expr y) { return functions::islessequal(x, y); } + + /// Comarison for less or greater. + /// \param x first operand + /// \param y second operand + /// \retval true if either less or greater + /// \retval false else +// template typename enable::type islessgreater(T x, U y) { return functions::islessgreater(x, y); } + inline bool islessgreater(half x, half y) { return functions::islessgreater(x, y); } + inline bool islessgreater(half x, expr y) { return functions::islessgreater(x, y); } + inline bool islessgreater(expr x, half y) { return functions::islessgreater(x, y); } + inline bool islessgreater(expr x, expr y) { return functions::islessgreater(x, y); } + + /// Check if unordered. + /// \param x first operand + /// \param y second operand + /// \retval true if unordered (one or two NaN operands) + /// \retval false else +// template typename enable::type isunordered(T x, U y) { return functions::isunordered(x, y); } + inline bool isunordered(half x, half y) { return functions::isunordered(x, y); } + inline bool isunordered(half x, expr y) { return functions::isunordered(x, y); } + inline bool isunordered(expr x, half y) { return functions::isunordered(x, y); } + inline bool isunordered(expr x, expr y) { return functions::isunordered(x, y); } + + /// \name Casting + /// \{ + + /// Cast to or from half-precision floating point number. + /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted + /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do. + /// It uses the default rounding mode. + /// + /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types + /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler + /// error and casting between [half](\ref half_float::half)s is just a no-op. + /// \tparam T destination type (half or built-in arithmetic type) + /// \tparam U source type (half or built-in arithmetic type) + /// \param arg value to cast + /// \return \a arg converted to destination type + template T half_cast(U arg) { return half_caster::cast(arg); } + + /// Cast to or from half-precision floating point number. + /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted + /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do. + /// + /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types + /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler + /// error and casting between [half](\ref half_float::half)s is just a no-op. + /// \tparam T destination type (half or built-in arithmetic type) + /// \tparam R rounding mode to use. + /// \tparam U source type (half or built-in arithmetic type) + /// \param arg value to cast + /// \return \a arg converted to destination type + template T half_cast(U arg) { return half_caster::cast(arg); } + /// \} + } + + using detail::operator==; + using detail::operator!=; + using detail::operator<; + using detail::operator>; + using detail::operator<=; + using detail::operator>=; + using detail::operator+; + using detail::operator-; + using detail::operator*; + using detail::operator/; + using detail::operator<<; + using detail::operator>>; + + using detail::abs; + using detail::fabs; + using detail::fmod; + using detail::remainder; + using detail::remquo; + using detail::fma; + using detail::fmax; + using detail::fmin; + using detail::fdim; + using detail::nanh; + using detail::exp; + using detail::expm1; + using detail::exp2; + using detail::log; + using detail::log10; + using detail::log1p; + using detail::log2; + using detail::sqrt; + using detail::cbrt; + using detail::hypot; + using detail::pow; + using detail::sin; + using detail::cos; + using detail::tan; + using detail::asin; + using detail::acos; + using detail::atan; + using detail::atan2; + using detail::sinh; + using detail::cosh; + using detail::tanh; + using detail::asinh; + using detail::acosh; + using detail::atanh; + using detail::erf; + using detail::erfc; + using detail::lgamma; + using detail::tgamma; + using detail::ceil; + using detail::floor; + using detail::trunc; + using detail::round; + using detail::lround; + using detail::nearbyint; + using detail::rint; + using detail::lrint; +#if HALF_ENABLE_CPP11_LONG_LONG + using detail::llround; + using detail::llrint; +#endif + using detail::frexp; + using detail::ldexp; + using detail::modf; + using detail::scalbn; + using detail::scalbln; + using detail::ilogb; + using detail::logb; + using detail::nextafter; + using detail::nexttoward; + using detail::copysign; + using detail::fpclassify; + using detail::isfinite; + using detail::isinf; + using detail::isnan; + using detail::isnormal; + using detail::signbit; + using detail::isgreater; + using detail::isgreaterequal; + using detail::isless; + using detail::islessequal; + using detail::islessgreater; + using detail::isunordered; + + using detail::half_cast; } -/// Fused multiply add. -/// \param x first operand -/// \param y second operand -/// \param z third operand -/// \return ( \a x * \a y ) + \a z rounded as one operation. -// template typename -//enable::type fma(T x, U y, V z) { return functions::fma(x, y, z); -//} -inline expr fma(half x, half y, half z) { return functions::fma(x, y, z); } -inline expr fma(half x, half y, expr z) { return functions::fma(x, y, z); } -inline expr fma(half x, expr y, half z) { return functions::fma(x, y, z); } -inline expr fma(half x, expr y, expr z) { return functions::fma(x, y, z); } -inline expr fma(expr x, half y, half z) { return functions::fma(x, y, z); } -inline expr fma(expr x, half y, expr z) { return functions::fma(x, y, z); } -inline expr fma(expr x, expr y, half z) { return functions::fma(x, y, z); } -inline expr fma(expr x, expr y, expr z) { return functions::fma(x, y, z); } - -/// Maximum of half expressions. -/// \param x first operand -/// \param y second operand -/// \return maximum of operands -// template typename result::type fmax(T -//x, U y) { return binary_specialized::fmax(x, y); } -inline half fmax(half x, half y) { - return binary_specialized::fmax(x, y); -} -inline expr fmax(half x, expr y) { - return binary_specialized::fmax(x, y); -} -inline expr fmax(expr x, half y) { - return binary_specialized::fmax(x, y); -} -inline expr fmax(expr x, expr y) { - return binary_specialized::fmax(x, y); -} -/// Minimum of half expressions. -/// \param x first operand -/// \param y second operand -/// \return minimum of operands -// template typename result::type fmin(T -//x, U y) { return binary_specialized::fmin(x, y); } -inline half fmin(half x, half y) { - return binary_specialized::fmin(x, y); -} -inline expr fmin(half x, expr y) { - return binary_specialized::fmin(x, y); -} -inline expr fmin(expr x, half y) { - return binary_specialized::fmin(x, y); -} -inline expr fmin(expr x, expr y) { - return binary_specialized::fmin(x, y); -} +/// Extensions to the C++ standard library. +namespace std +{ + /// Numeric limits for half-precision floats. + /// Because of the underlying single-precision implementation of many operations, it inherits some properties from + /// `std::numeric_limits`. + template<> class numeric_limits : public numeric_limits + { + public: + /// Supports signed values. + static HALF_CONSTEXPR_CONST bool is_signed = true; -/// Positive difference. -/// \param x first operand -/// \param y second operand -/// \return \a x - \a y or 0 if difference negative -// template typename enable::type -//fdim(T x, U y) { return functions::fdim(x, y); } -inline expr fdim(half x, half y) { return functions::fdim(x, y); } -inline expr fdim(half x, expr y) { return functions::fdim(x, y); } -inline expr fdim(expr x, half y) { return functions::fdim(x, y); } -inline expr fdim(expr x, expr y) { return functions::fdim(x, y); } - -/// Get NaN value. -/// \return quiet NaN -inline half nanh(const char *) { return functions::nanh(); } - -/// \} -/// \name Exponential functions -/// \{ - -/// Exponential function. -/// \param arg function argument -/// \return e raised to \a arg -// template typename enable::type exp(T arg) { -//return functions::exp(arg); } -inline expr exp(half arg) { return functions::exp(arg); } -inline expr exp(expr arg) { return functions::exp(arg); } - -/// Exponential minus one. -/// \param arg function argument -/// \return e raised to \a arg subtracted by 1 -// template typename enable::type expm1(T arg) { -//return functions::expm1(arg); } -inline expr expm1(half arg) { return functions::expm1(arg); } -inline expr expm1(expr arg) { return functions::expm1(arg); } - -/// Binary exponential. -/// \param arg function argument -/// \return 2 raised to \a arg -// template typename enable::type exp2(T arg) { -//return functions::exp2(arg); } -inline expr exp2(half arg) { return functions::exp2(arg); } -inline expr exp2(expr arg) { return functions::exp2(arg); } - -/// Natural logorithm. -/// \param arg function argument -/// \return logarithm of \a arg to base e -// template typename enable::type log(T arg) { -//return functions::log(arg); } -inline expr log(half arg) { return functions::log(arg); } -inline expr log(expr arg) { return functions::log(arg); } - -/// Common logorithm. -/// \param arg function argument -/// \return logarithm of \a arg to base 10 -// template typename enable::type log10(T arg) { -//return functions::log10(arg); } -inline expr log10(half arg) { return functions::log10(arg); } -inline expr log10(expr arg) { return functions::log10(arg); } - -/// Natural logorithm. -/// \param arg function argument -/// \return logarithm of \a arg plus 1 to base e -// template typename enable::type log1p(T arg) { -//return functions::log1p(arg); } -inline expr log1p(half arg) { return functions::log1p(arg); } -inline expr log1p(expr arg) { return functions::log1p(arg); } - -/// Binary logorithm. -/// \param arg function argument -/// \return logarithm of \a arg to base 2 -// template typename enable::type log2(T arg) { -//return functions::log2(arg); } -inline expr log2(half arg) { return functions::log2(arg); } -inline expr log2(expr arg) { return functions::log2(arg); } - -/// \} -/// \name Power functions -/// \{ - -/// Square root. -/// \param arg function argument -/// \return square root of \a arg -// template typename enable::type sqrt(T arg) { -//return functions::sqrt(arg); } -inline expr sqrt(half arg) { return functions::sqrt(arg); } -inline expr sqrt(expr arg) { return functions::sqrt(arg); } - -/// Cubic root. -/// \param arg function argument -/// \return cubic root of \a arg -// template typename enable::type cbrt(T arg) { -//return functions::cbrt(arg); } -inline expr cbrt(half arg) { return functions::cbrt(arg); } -inline expr cbrt(expr arg) { return functions::cbrt(arg); } - -/// Hypotenuse function. -/// \param x first argument -/// \param y second argument -/// \return square root of sum of squares without internal over- or underflows -// template typename enable::type -//hypot(T x, U y) { return functions::hypot(x, y); } -inline expr hypot(half x, half y) { return functions::hypot(x, y); } -inline expr hypot(half x, expr y) { return functions::hypot(x, y); } -inline expr hypot(expr x, half y) { return functions::hypot(x, y); } -inline expr hypot(expr x, expr y) { return functions::hypot(x, y); } - -/// Power function. -/// \param base first argument -/// \param exp second argument -/// \return \a base raised to \a exp -// template typename enable::type -//pow(T base, U exp) { return functions::pow(base, exp); } -inline expr pow(half base, half exp) { return functions::pow(base, exp); } -inline expr pow(half base, expr exp) { return functions::pow(base, exp); } -inline expr pow(expr base, half exp) { return functions::pow(base, exp); } -inline expr pow(expr base, expr exp) { return functions::pow(base, exp); } - -/// \} -/// \name Trigonometric functions -/// \{ - -/// Sine function. -/// \param arg function argument -/// \return sine value of \a arg -// template typename enable::type sin(T arg) { -//return functions::sin(arg); } -inline expr sin(half arg) { return functions::sin(arg); } -inline expr sin(expr arg) { return functions::sin(arg); } - -/// Cosine function. -/// \param arg function argument -/// \return cosine value of \a arg -// template typename enable::type cos(T arg) { -//return functions::cos(arg); } -inline expr cos(half arg) { return functions::cos(arg); } -inline expr cos(expr arg) { return functions::cos(arg); } - -/// Tangent function. -/// \param arg function argument -/// \return tangent value of \a arg -// template typename enable::type tan(T arg) { -//return functions::tan(arg); } -inline expr tan(half arg) { return functions::tan(arg); } -inline expr tan(expr arg) { return functions::tan(arg); } - -/// Arc sine. -/// \param arg function argument -/// \return arc sine value of \a arg -// template typename enable::type asin(T arg) { -//return functions::asin(arg); } -inline expr asin(half arg) { return functions::asin(arg); } -inline expr asin(expr arg) { return functions::asin(arg); } - -/// Arc cosine function. -/// \param arg function argument -/// \return arc cosine value of \a arg -// template typename enable::type acos(T arg) { -//return functions::acos(arg); } -inline expr acos(half arg) { return functions::acos(arg); } -inline expr acos(expr arg) { return functions::acos(arg); } - -/// Arc tangent function. -/// \param arg function argument -/// \return arc tangent value of \a arg -// template typename enable::type atan(T arg) { -//return functions::atan(arg); } -inline expr atan(half arg) { return functions::atan(arg); } -inline expr atan(expr arg) { return functions::atan(arg); } - -/// Arc tangent function. -/// \param x first argument -/// \param y second argument -/// \return arc tangent value -// template typename enable::type -//atan2(T x, U y) { return functions::atan2(x, y); } -inline expr atan2(half x, half y) { return functions::atan2(x, y); } -inline expr atan2(half x, expr y) { return functions::atan2(x, y); } -inline expr atan2(expr x, half y) { return functions::atan2(x, y); } -inline expr atan2(expr x, expr y) { return functions::atan2(x, y); } - -/// \} -/// \name Hyperbolic functions -/// \{ - -/// Hyperbolic sine. -/// \param arg function argument -/// \return hyperbolic sine value of \a arg -// template typename enable::type sinh(T arg) { -//return functions::sinh(arg); } -inline expr sinh(half arg) { return functions::sinh(arg); } -inline expr sinh(expr arg) { return functions::sinh(arg); } - -/// Hyperbolic cosine. -/// \param arg function argument -/// \return hyperbolic cosine value of \a arg -// template typename enable::type cosh(T arg) { -//return functions::cosh(arg); } -inline expr cosh(half arg) { return functions::cosh(arg); } -inline expr cosh(expr arg) { return functions::cosh(arg); } - -/// Hyperbolic tangent. -/// \param arg function argument -/// \return hyperbolic tangent value of \a arg -// template typename enable::type tanh(T arg) { -//return functions::tanh(arg); } -inline expr tanh(half arg) { return functions::tanh(arg); } -inline expr tanh(expr arg) { return functions::tanh(arg); } - -/// Hyperbolic area sine. -/// \param arg function argument -/// \return area sine value of \a arg -// template typename enable::type asinh(T arg) { -//return functions::asinh(arg); } -inline expr asinh(half arg) { return functions::asinh(arg); } -inline expr asinh(expr arg) { return functions::asinh(arg); } - -/// Hyperbolic area cosine. -/// \param arg function argument -/// \return area cosine value of \a arg -// template typename enable::type acosh(T arg) { -//return functions::acosh(arg); } -inline expr acosh(half arg) { return functions::acosh(arg); } -inline expr acosh(expr arg) { return functions::acosh(arg); } - -/// Hyperbolic area tangent. -/// \param arg function argument -/// \return area tangent value of \a arg -// template typename enable::type atanh(T arg) { -//return functions::atanh(arg); } -inline expr atanh(half arg) { return functions::atanh(arg); } -inline expr atanh(expr arg) { return functions::atanh(arg); } - -/// \} -/// \name Error and gamma functions -/// \{ - -/// Error function. -/// \param arg function argument -/// \return error function value of \a arg -// template typename enable::type erf(T arg) { -//return functions::erf(arg); } -inline expr erf(half arg) { return functions::erf(arg); } -inline expr erf(expr arg) { return functions::erf(arg); } - -/// Complementary error function. -/// \param arg function argument -/// \return 1 minus error function value of \a arg -// template typename enable::type erfc(T arg) { -//return functions::erfc(arg); } -inline expr erfc(half arg) { return functions::erfc(arg); } -inline expr erfc(expr arg) { return functions::erfc(arg); } - -/// Natural logarithm of gamma function. -/// \param arg function argument -/// \return natural logarith of gamma function for \a arg -// template typename enable::type lgamma(T arg) { -//return functions::lgamma(arg); } -inline expr lgamma(half arg) { return functions::lgamma(arg); } -inline expr lgamma(expr arg) { return functions::lgamma(arg); } - -/// Gamma function. -/// \param arg function argument -/// \return gamma function value of \a arg -// template typename enable::type tgamma(T arg) { -//return functions::tgamma(arg); } -inline expr tgamma(half arg) { return functions::tgamma(arg); } -inline expr tgamma(expr arg) { return functions::tgamma(arg); } - -/// \} -/// \name Rounding -/// \{ - -/// Nearest integer not less than half value. -/// \param arg half to round -/// \return nearest integer not less than \a arg -// template typename enable::type ceil(T arg) { -//return functions::ceil(arg); } -inline half ceil(half arg) { return functions::ceil(arg); } -inline half ceil(expr arg) { return functions::ceil(arg); } - -/// Nearest integer not greater than half value. -/// \param arg half to round -/// \return nearest integer not greater than \a arg -// template typename enable::type floor(T arg) { -//return functions::floor(arg); } -inline half floor(half arg) { return functions::floor(arg); } -inline half floor(expr arg) { return functions::floor(arg); } - -/// Nearest integer not greater in magnitude than half value. -/// \param arg half to round -/// \return nearest integer not greater in magnitude than \a arg -// template typename enable::type trunc(T arg) { -//return functions::trunc(arg); } -inline half trunc(half arg) { return functions::trunc(arg); } -inline half trunc(expr arg) { return functions::trunc(arg); } - -/// Nearest integer. -/// \param arg half to round -/// \return nearest integer, rounded away from zero in half-way cases -// template typename enable::type round(T arg) { -//return functions::round(arg); } -inline half round(half arg) { return functions::round(arg); } -inline half round(expr arg) { return functions::round(arg); } - -/// Nearest integer. -/// \param arg half to round -/// \return nearest integer, rounded away from zero in half-way cases -// template typename enable::type lround(T arg) { -//return functions::lround(arg); } -inline long lround(half arg) { return functions::lround(arg); } -inline long lround(expr arg) { return functions::lround(arg); } - -/// Nearest integer using half's internal rounding mode. -/// \param arg half expression to round -/// \return nearest integer using default rounding mode -// template typename enable::type nearbyint(T -//arg) { return functions::nearbyint(arg); } -inline half nearbyint(half arg) { return functions::rint(arg); } -inline half nearbyint(expr arg) { return functions::rint(arg); } - -/// Nearest integer using half's internal rounding mode. -/// \param arg half expression to round -/// \return nearest integer using default rounding mode -// template typename enable::type rint(T arg) { -//return functions::rint(arg); } -inline half rint(half arg) { return functions::rint(arg); } -inline half rint(expr arg) { return functions::rint(arg); } - -/// Nearest integer using half's internal rounding mode. -/// \param arg half expression to round -/// \return nearest integer using default rounding mode -// template typename enable::type lrint(T arg) { -//return functions::lrint(arg); } -inline long lrint(half arg) { return functions::lrint(arg); } -inline long lrint(expr arg) { return functions::lrint(arg); } -#if HALF_ENABLE_CPP11_LONG_LONG -/// Nearest integer. -/// \param arg half to round -/// \return nearest integer, rounded away from zero in half-way cases -// template typename enable::type llround(T -//arg) { return functions::llround(arg); } -inline long long llround(half arg) { return functions::llround(arg); } -inline long long llround(expr arg) { return functions::llround(arg); } - -/// Nearest integer using half's internal rounding mode. -/// \param arg half expression to round -/// \return nearest integer using default rounding mode -// template typename enable::type llrint(T -//arg) { return functions::llrint(arg); } -inline long long llrint(half arg) { return functions::llrint(arg); } -inline long long llrint(expr arg) { return functions::llrint(arg); } -#endif + /// Is not exact. + static HALF_CONSTEXPR_CONST bool is_exact = false; -/// \} -/// \name Floating point manipulation -/// \{ - -/// Decompress floating point number. -/// \param arg number to decompress -/// \param exp address to store exponent at -/// \return significant in range [0.5, 1) -// template typename enable::type frexp(T arg, -//int *exp) { return functions::frexp(arg, exp); } -inline half frexp(half arg, int *exp) { return functions::frexp(arg, exp); } -inline half frexp(expr arg, int *exp) { return functions::frexp(arg, exp); } - -/// Multiply by power of two. -/// \param arg number to modify -/// \param exp power of two to multiply with -/// \return \a arg multplied by 2 raised to \a exp -// template typename enable::type ldexp(T arg, -//int exp) { return functions::scalbln(arg, exp); } -inline half ldexp(half arg, int exp) { return functions::scalbln(arg, exp); } -inline half ldexp(expr arg, int exp) { return functions::scalbln(arg, exp); } - -/// Extract integer and fractional parts. -/// \param arg number to decompress -/// \param iptr address to store integer part at -/// \return fractional part -// template typename enable::type modf(T arg, -//half *iptr) { return functions::modf(arg, iptr); } -inline half modf(half arg, half *iptr) { return functions::modf(arg, iptr); } -inline half modf(expr arg, half *iptr) { return functions::modf(arg, iptr); } - -/// Multiply by power of two. -/// \param arg number to modify -/// \param exp power of two to multiply with -/// \return \a arg multplied by 2 raised to \a exp -// template typename enable::type scalbn(T arg, -//int exp) { return functions::scalbln(arg, exp); } -inline half scalbn(half arg, int exp) { return functions::scalbln(arg, exp); } -inline half scalbn(expr arg, int exp) { return functions::scalbln(arg, exp); } - -/// Multiply by power of two. -/// \param arg number to modify -/// \param exp power of two to multiply with -/// \return \a arg multplied by 2 raised to \a exp -// template typename enable::type scalbln(T arg, -//long exp) { return functions::scalbln(arg, exp); } -inline half scalbln(half arg, long exp) { return functions::scalbln(arg, exp); } -inline half scalbln(expr arg, long exp) { return functions::scalbln(arg, exp); } - -/// Extract exponent. -/// \param arg number to query -/// \return floating point exponent -/// \retval FP_ILOGB0 for zero -/// \retval FP_ILOGBNAN for NaN -/// \retval MAX_INT for infinity -// template typename enable::type ilogb(T arg) { -//return functions::ilogb(arg); } -inline int ilogb(half arg) { return functions::ilogb(arg); } -inline int ilogb(expr arg) { return functions::ilogb(arg); } - -/// Extract exponent. -/// \param arg number to query -/// \return floating point exponent -// template typename enable::type logb(T arg) { -//return functions::logb(arg); } -inline half logb(half arg) { return functions::logb(arg); } -inline half logb(expr arg) { return functions::logb(arg); } - -/// Next representable value. -/// \param from value to compute next representable value for -/// \param to direction towards which to compute next value -/// \return next representable value after \a from in direction towards \a to -// template typename enable::type -//nextafter(T from, U to) { return functions::nextafter(from, to); } -inline half nextafter(half from, half to) { - return functions::nextafter(from, to); -} -inline half nextafter(half from, expr to) { - return functions::nextafter(from, to); -} -inline half nextafter(expr from, half to) { - return functions::nextafter(from, to); -} -inline half nextafter(expr from, expr to) { - return functions::nextafter(from, to); -} + /// Doesn't provide modulo arithmetic. + static HALF_CONSTEXPR_CONST bool is_modulo = false; -/// Next representable value. -/// \param from value to compute next representable value for -/// \param to direction towards which to compute next value -/// \return next representable value after \a from in direction towards \a to -// template typename enable::type nexttoward(T -//from, long double to) { return functions::nexttoward(from, to); } -inline half nexttoward(half from, long double to) { - return functions::nexttoward(from, to); -} -inline half nexttoward(expr from, long double to) { - return functions::nexttoward(from, to); -} + /// IEEE conformant. + static HALF_CONSTEXPR_CONST bool is_iec559 = true; -/// Take sign. -/// \param x value to change sign for -/// \param y value to take sign from -/// \return value equal to \a x in magnitude and to \a y in sign -// template typename enable::type -//copysign(T x, U y) { return functions::copysign(x, y); } -inline half copysign(half x, half y) { return functions::copysign(x, y); } -inline half copysign(half x, expr y) { return functions::copysign(x, y); } -inline half copysign(expr x, half y) { return functions::copysign(x, y); } -inline half copysign(expr x, expr y) { return functions::copysign(x, y); } - -/// \} -/// \name Floating point classification -/// \{ - -/// Classify floating point value. -/// \param arg number to classify -/// \retval FP_ZERO for positive and negative zero -/// \retval FP_SUBNORMAL for subnormal numbers -/// \retval FP_INFINITY for positive and negative infinity -/// \retval FP_NAN for NaNs -/// \retval FP_NORMAL for all other (normal) values -// template typename enable::type fpclassify(T -//arg) { return functions::fpclassify(arg); } -inline int fpclassify(half arg) { return functions::fpclassify(arg); } -inline int fpclassify(expr arg) { return functions::fpclassify(arg); } - -/// Check if finite number. -/// \param arg number to check -/// \retval true if neither infinity nor NaN -/// \retval false else -// template typename enable::type isfinite(T arg) -//{ return functions::isfinite(arg); } -inline bool isfinite(half arg) { return functions::isfinite(arg); } -inline bool isfinite(expr arg) { return functions::isfinite(arg); } - -/// Check for infinity. -/// \param arg number to check -/// \retval true for positive or negative infinity -/// \retval false else -// template typename enable::type isinf(T arg) { -//return functions::isinf(arg); } -inline bool isinf(half arg) { return functions::isinf(arg); } -inline bool isinf(expr arg) { return functions::isinf(arg); } - -/// Check for NaN. -/// \param arg number to check -/// \retval true for NaNs -/// \retval false else -// template typename enable::type isnan(T arg) { -//return functions::isnan(arg); } -inline bool isnan(half arg) { return functions::isnan(arg); } -inline bool isnan(expr arg) { return functions::isnan(arg); } - -/// Check if normal number. -/// \param arg number to check -/// \retval true if normal number -/// \retval false if either subnormal, zero, infinity or NaN -// template typename enable::type isnormal(T arg) -//{ return functions::isnormal(arg); } -inline bool isnormal(half arg) { return functions::isnormal(arg); } -inline bool isnormal(expr arg) { return functions::isnormal(arg); } - -/// Check sign. -/// \param arg number to check -/// \retval true for negative number -/// \retval false for positive number -// template typename enable::type signbit(T arg) -//{ return functions::signbit(arg); } -inline bool signbit(half arg) { return functions::signbit(arg); } -inline bool signbit(expr arg) { return functions::signbit(arg); } - -/// \} -/// \name Comparison -/// \{ - -/// Comparison for greater than. -/// \param x first operand -/// \param y second operand -/// \retval true if \a x greater than \a y -/// \retval false else -// template typename enable::type -//isgreater(T x, U y) { return functions::isgreater(x, y); } -inline bool isgreater(half x, half y) { return functions::isgreater(x, y); } -inline bool isgreater(half x, expr y) { return functions::isgreater(x, y); } -inline bool isgreater(expr x, half y) { return functions::isgreater(x, y); } -inline bool isgreater(expr x, expr y) { return functions::isgreater(x, y); } - -/// Comparison for greater equal. -/// \param x first operand -/// \param y second operand -/// \retval true if \a x greater equal \a y -/// \retval false else -// template typename enable::type -//isgreaterequal(T x, U y) { return functions::isgreaterequal(x, y); } -inline bool isgreaterequal(half x, half y) { - return functions::isgreaterequal(x, y); -} -inline bool isgreaterequal(half x, expr y) { - return functions::isgreaterequal(x, y); -} -inline bool isgreaterequal(expr x, half y) { - return functions::isgreaterequal(x, y); -} -inline bool isgreaterequal(expr x, expr y) { - return functions::isgreaterequal(x, y); -} + /// Supports infinity. + static HALF_CONSTEXPR_CONST bool has_infinity = true; -/// Comparison for less than. -/// \param x first operand -/// \param y second operand -/// \retval true if \a x less than \a y -/// \retval false else -// template typename enable::type -//isless(T x, U y) { return functions::isless(x, y); } -inline bool isless(half x, half y) { return functions::isless(x, y); } -inline bool isless(half x, expr y) { return functions::isless(x, y); } -inline bool isless(expr x, half y) { return functions::isless(x, y); } -inline bool isless(expr x, expr y) { return functions::isless(x, y); } - -/// Comparison for less equal. -/// \param x first operand -/// \param y second operand -/// \retval true if \a x less equal \a y -/// \retval false else -// template typename enable::type -//islessequal(T x, U y) { return functions::islessequal(x, y); } -inline bool islessequal(half x, half y) { return functions::islessequal(x, y); } -inline bool islessequal(half x, expr y) { return functions::islessequal(x, y); } -inline bool islessequal(expr x, half y) { return functions::islessequal(x, y); } -inline bool islessequal(expr x, expr y) { return functions::islessequal(x, y); } - -/// Comarison for less or greater. -/// \param x first operand -/// \param y second operand -/// \retval true if either less or greater -/// \retval false else -// template typename enable::type -//islessgreater(T x, U y) { return functions::islessgreater(x, y); } -inline bool islessgreater(half x, half y) { - return functions::islessgreater(x, y); -} -inline bool islessgreater(half x, expr y) { - return functions::islessgreater(x, y); -} -inline bool islessgreater(expr x, half y) { - return functions::islessgreater(x, y); -} -inline bool islessgreater(expr x, expr y) { - return functions::islessgreater(x, y); -} + /// Supports quiet NaNs. + static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true; -/// Check if unordered. -/// \param x first operand -/// \param y second operand -/// \retval true if unordered (one or two NaN operands) -/// \retval false else -// template typename enable::type -//isunordered(T x, U y) { return functions::isunordered(x, y); } -inline bool isunordered(half x, half y) { return functions::isunordered(x, y); } -inline bool isunordered(half x, expr y) { return functions::isunordered(x, y); } -inline bool isunordered(expr x, half y) { return functions::isunordered(x, y); } -inline bool isunordered(expr x, expr y) { return functions::isunordered(x, y); } - -/// \name Casting -/// \{ - -/// Cast to or from half-precision floating point number. -/// This casts between [half](\ref half_float::half) and any built-in arithmetic -/// type. The values are converted -/// directly using the given rounding mode, without any roundtrip over `float` -/// that a `static_cast` would otherwise do. -/// It uses the default rounding mode. -/// -/// Using this cast with neither of the two types being a [half](\ref -/// half_float::half) or with any of the two types -/// not being a built-in arithmetic type (apart from [half](\ref -/// half_float::half), of course) results in a compiler -/// error and casting between [half](\ref half_float::half)s is just a no-op. -/// \tparam T destination type (half or built-in arithmetic type) -/// \tparam U source type (half or built-in arithmetic type) -/// \param arg value to cast -/// \return \a arg converted to destination type -template -T half_cast(U arg) { - return half_caster::cast(arg); -} + /// Supports subnormal values. + static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present; -/// Cast to or from half-precision floating point number. -/// This casts between [half](\ref half_float::half) and any built-in arithmetic -/// type. The values are converted -/// directly using the given rounding mode, without any roundtrip over `float` -/// that a `static_cast` would otherwise do. -/// -/// Using this cast with neither of the two types being a [half](\ref -/// half_float::half) or with any of the two types -/// not being a built-in arithmetic type (apart from [half](\ref -/// half_float::half), of course) results in a compiler -/// error and casting between [half](\ref half_float::half)s is just a no-op. -/// \tparam T destination type (half or built-in arithmetic type) -/// \tparam R rounding mode to use. -/// \tparam U source type (half or built-in arithmetic type) -/// \param arg value to cast -/// \return \a arg converted to destination type -template -T half_cast(U arg) { - return half_caster::cast(arg); -} -/// \} -} + /// Rounding mode. + /// Due to the mix of internal single-precision computations (using the rounding mode of the underlying + /// single-precision implementation) with the rounding mode of the single-to-half conversions, the actual rounding + /// mode might be `std::round_indeterminate` if the default half-precision rounding mode doesn't match the + /// single-precision rounding mode. + static HALF_CONSTEXPR_CONST float_round_style round_style = (std::numeric_limits::round_style== + half_float::half::round_style) ? half_float::half::round_style : round_indeterminate; -using detail::operator==; -using detail::operator!=; -using detail::operator<; -using detail::operator>; -using detail::operator<=; -using detail::operator>=; -using detail::operator+; -using detail::operator-; -using detail::operator*; -using detail::operator/; -using detail::operator<<; -using detail::operator>>; - -using detail::abs; -using detail::fabs; -using detail::fmod; -using detail::remainder; -using detail::remquo; -using detail::fma; -using detail::fmax; -using detail::fmin; -using detail::fdim; -using detail::nanh; -using detail::exp; -using detail::expm1; -using detail::exp2; -using detail::log; -using detail::log10; -using detail::log1p; -using detail::log2; -using detail::sqrt; -using detail::cbrt; -using detail::hypot; -using detail::pow; -using detail::sin; -using detail::cos; -using detail::tan; -using detail::asin; -using detail::acos; -using detail::atan; -using detail::atan2; -using detail::sinh; -using detail::cosh; -using detail::tanh; -using detail::asinh; -using detail::acosh; -using detail::atanh; -using detail::erf; -using detail::erfc; -using detail::lgamma; -using detail::tgamma; -using detail::ceil; -using detail::floor; -using detail::trunc; -using detail::round; -using detail::lround; -using detail::nearbyint; -using detail::rint; -using detail::lrint; -#if HALF_ENABLE_CPP11_LONG_LONG -using detail::llround; -using detail::llrint; -#endif -using detail::frexp; -using detail::ldexp; -using detail::modf; -using detail::scalbn; -using detail::scalbln; -using detail::ilogb; -using detail::logb; -using detail::nextafter; -using detail::nexttoward; -using detail::copysign; -using detail::fpclassify; -using detail::isfinite; -using detail::isinf; -using detail::isnan; -using detail::isnormal; -using detail::signbit; -using detail::isgreater; -using detail::isgreaterequal; -using detail::isless; -using detail::islessequal; -using detail::islessgreater; -using detail::isunordered; - -using detail::half_cast; -} + /// Significant digits. + static HALF_CONSTEXPR_CONST int digits = 11; -/// Extensions to the C++ standard library. -namespace std { -/// Numeric limits for half-precision floats. -/// Because of the underlying single-precision implementation of many -/// operations, it inherits some properties from -/// `std::numeric_limits`. -template <> -class numeric_limits : public numeric_limits { - public: - /// Supports signed values. - static HALF_CONSTEXPR_CONST bool is_signed = true; - - /// Is not exact. - static HALF_CONSTEXPR_CONST bool is_exact = false; - - /// Doesn't provide modulo arithmetic. - static HALF_CONSTEXPR_CONST bool is_modulo = false; - - /// IEEE conformant. - static HALF_CONSTEXPR_CONST bool is_iec559 = true; - - /// Supports infinity. - static HALF_CONSTEXPR_CONST bool has_infinity = true; - - /// Supports quiet NaNs. - static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true; - - /// Supports subnormal values. - static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present; - - /// Rounding mode. - /// Due to the mix of internal single-precision computations (using the - /// rounding mode of the underlying - /// single-precision implementation) with the rounding mode of the - /// single-to-half conversions, the actual rounding - /// mode might be `std::round_indeterminate` if the default half-precision - /// rounding mode doesn't match the - /// single-precision rounding mode. - static HALF_CONSTEXPR_CONST float_round_style round_style = - (std::numeric_limits::round_style == half_float::half::round_style) - ? half_float::half::round_style - : round_indeterminate; - - /// Significant digits. - static HALF_CONSTEXPR_CONST int digits = 11; - - /// Significant decimal digits. - static HALF_CONSTEXPR_CONST int digits10 = 3; - - /// Required decimal digits to represent all possible values. - static HALF_CONSTEXPR_CONST int max_digits10 = 5; - - /// Number base. - static HALF_CONSTEXPR_CONST int radix = 2; - - /// One more than smallest exponent. - static HALF_CONSTEXPR_CONST int min_exponent = -13; - - /// Smallest normalized representable power of 10. - static HALF_CONSTEXPR_CONST int min_exponent10 = -4; - - /// One more than largest exponent - static HALF_CONSTEXPR_CONST int max_exponent = 16; - - /// Largest finitely representable power of 10. - static HALF_CONSTEXPR_CONST int max_exponent10 = 4; - - /// Smallest positive normal value. - static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW { - return half_float::half(half_float::detail::binary, 0x0400); - } - - /// Smallest finite value. - static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW { - return half_float::half(half_float::detail::binary, 0xFBFF); - } - - /// Largest finite value. - static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW { - return half_float::half(half_float::detail::binary, 0x7BFF); - } - - /// Difference between one and next representable value. - static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW { - return half_float::half(half_float::detail::binary, 0x1400); - } - - /// Maximum rounding error. - static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW { - return half_float::half( - half_float::detail::binary, - (round_style == std::round_to_nearest) ? 0x3800 : 0x3C00); - } - - /// Positive infinity. - static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW { - return half_float::half(half_float::detail::binary, 0x7C00); - } - - /// Quiet NaN. - static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW { - return half_float::half(half_float::detail::binary, 0x7FFF); - } - - /// Signalling NaN. - static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW { - return half_float::half(half_float::detail::binary, 0x7DFF); - } - - /// Smallest positive subnormal value. - static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW { - return half_float::half(half_float::detail::binary, 0x0001); - } -}; + /// Significant decimal digits. + static HALF_CONSTEXPR_CONST int digits10 = 3; + + /// Required decimal digits to represent all possible values. + static HALF_CONSTEXPR_CONST int max_digits10 = 5; + + /// Number base. + static HALF_CONSTEXPR_CONST int radix = 2; + + /// One more than smallest exponent. + static HALF_CONSTEXPR_CONST int min_exponent = -13; + + /// Smallest normalized representable power of 10. + static HALF_CONSTEXPR_CONST int min_exponent10 = -4; + + /// One more than largest exponent + static HALF_CONSTEXPR_CONST int max_exponent = 16; + + /// Largest finitely representable power of 10. + static HALF_CONSTEXPR_CONST int max_exponent10 = 4; + + /// Smallest positive normal value. + static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0400); } + + /// Smallest finite value. + static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0xFBFF); } + + /// Largest finite value. + static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7BFF); } + + /// Difference between one and next representable value. + static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x1400); } + + /// Maximum rounding error. + static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW + { return half_float::half(half_float::detail::binary, (round_style==std::round_to_nearest) ? 0x3800 : 0x3C00); } + + /// Positive infinity. + static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7C00); } + + /// Quiet NaN. + static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7FFF); } + + /// Signalling NaN. + static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7DFF); } + + /// Smallest positive subnormal value. + static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0001); } + }; #if HALF_ENABLE_CPP11_HASH -/// Hash function for half-precision floats. -/// This is only defined if C++11 `std::hash` is supported and enabled. -template <> -struct hash //: unary_function -{ - /// Type of function argument. - typedef half_float::half argument_type; - - /// Function return type. - typedef size_t result_type; - - /// Compute hash function. - /// \param arg half to hash - /// \return hash value - result_type operator()(argument_type arg) const { - return hash()(static_cast(arg.data_) & - -(arg.data_ != 0x8000)); - } -}; + /// Hash function for half-precision floats. + /// This is only defined if C++11 `std::hash` is supported and enabled. + template<> struct hash //: unary_function + { + /// Type of function argument. + typedef half_float::half argument_type; + + /// Function return type. + typedef size_t result_type; + + /// Compute hash function. + /// \param arg half to hash + /// \return hash value + result_type operator()(argument_type arg) const + { return hash()(static_cast(arg.data_)&-(arg.data_!=0x8000)); } + }; #endif } + #undef HALF_CONSTEXPR #undef HALF_CONSTEXPR_CONST #undef HALF_NOEXCEPT #undef HALF_NOTHROW #ifdef HALF_POP_WARNINGS -#pragma warning(pop) -#undef HALF_POP_WARNINGS + #pragma warning(pop) + #undef HALF_POP_WARNINGS #endif #endif diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 861f3bf..014e588 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -8,16 +8,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -27,34 +25,34 @@ // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#include "instructions.h" #include "half.h" #include "half.hpp" -#include "opcodes.h" +#include "instructions.h" #include "ptx_ir.h" +#include "opcodes.h" #include "ptx_sim.h" -typedef void *yyscan_t; +typedef void * yyscan_t; class ptx_recognizer; -#include -#include -#include -#include -#include +#include "ptx.tab.h" #include -#include +#include #include -#include -#include -#include +#include +#include "cuda-math.h" #include "../abstract_hardware_model.h" +#include "ptx_loader.h" +#include "cuda_device_printf.h" #include "../gpgpu-sim/gpu-sim.h" #include "../gpgpu-sim/shader.h" -#include "cuda-math.h" -#include "cuda_device_printf.h" -#include "ptx.tab.h" -#include "ptx_loader.h" +#include +#include +#include +#include +#include +#include +#include -// Jin: include device runtime for CDP +//Jin: include device runtime for CDP #include "cuda_device_runtime.h" #include @@ -62,6425 +60,5629 @@ class ptx_recognizer; using half_float::half; + + const char *g_opcode_string[NUM_OPCODES] = { -#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) STR, -#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) STR, +#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR, +#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR, #include "opcodes.def" #undef OP_DEF #undef OP_W_DEF }; -// Using profiled information::check the TensorCoreMatrixArrangement.xls for -// details -unsigned thread_group_offset(int thread, unsigned wmma_type, - unsigned wmma_layout, unsigned type, int stride) { - unsigned offset; - unsigned load_a_row[8] = {0, 128, 0, 128, 64, 192, 64, 192}; - unsigned load_a_col[8] = {0, 8, 0, 8, 4, 12, 4, 12}; - unsigned load_b_row[8] = {0, 8, 0, 8, 4, 12, 4, 12}; - unsigned load_b_col[8] = {0, 128, 0, 128, 64, 192, 64, 192}; - unsigned load_c_float_row[8] = {0, 128, 8, 136, 64, 192, 72, 200}; - unsigned load_c_float_col[8] = {0, 8, 128, 136, 4, 12, 132, 140}; - unsigned load_c_half_row[8] = {0, 128, 8, 136, 64, 192, 72, 200}; - unsigned load_c_half_col[8] = {0, 8, 128, 136, 4, 12, 132, 140}; - unsigned thread_group = thread / 4; - unsigned in_tg_index = thread % 4; - - switch (wmma_type) { - case LOAD_A: - if (wmma_layout == ROW) - offset = load_a_row[thread_group] + 16 * in_tg_index; - else - offset = load_a_col[thread_group] + 16 * in_tg_index; - break; - - case LOAD_B: - if (wmma_layout == ROW) - offset = load_b_row[thread_group] + 16 * in_tg_index; - else - offset = load_b_col[thread_group] + 16 * in_tg_index; - break; - - case LOAD_C: - case STORE_D: - if (type == F16_TYPE) { - if (wmma_layout == ROW) - offset = load_c_half_row[thread_group] + 16 * in_tg_index; - else - offset = load_c_half_col[thread_group] + in_tg_index; - } else { - if (wmma_layout == ROW) - offset = load_c_float_row[thread_group]; - else - offset = load_c_float_col[thread_group]; - - switch (in_tg_index) { - case 0: - break; - case 1: - if (wmma_layout == ROW) - offset += 16; - else - offset += 1; - break; - case 2: - if (wmma_layout == ROW) - offset += 2; - else - offset += 32; - break; - case 3: - if (wmma_layout == ROW) - offset += 18; - else - offset += 33; - break; - default: - abort(); - } - } - break; - - default: - abort(); - } - offset = (offset / 16) * stride + offset % 16; - return offset; -} - -int acc_float_offset(int index, int wmma_layout, int stride) { - int c_row_offset[] = {0, 1, 32, 33, 4, 5, 36, 37}; - int c_col_offset[] = {0, 16, 2, 18, 64, 80, 66, 82}; - int offset; - - if (wmma_layout == ROW) - offset = c_row_offset[index]; - else if (wmma_layout == COL) - offset = c_col_offset[index]; - else { - printf("wrong layout"); - abort(); - } - offset = (offset / 16) * stride + offset % 16; - return offset; -} - -void inst_not_implemented(const ptx_instruction *pI); -ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, - operand_info dstInfo, unsigned type, - ptx_thread_info *thread); - -void sign_extend(ptx_reg_t &data, unsigned src_size, const operand_info &dst); - -void ptx_thread_info::set_reg(const symbol *reg, const ptx_reg_t &value) { - assert(reg != NULL); - if (reg->name() == "_") return; - assert(!m_regs.empty()); - assert(reg->uid() > 0); - m_regs.back()[reg] = value; - if (m_enable_debug_trace) m_debug_trace_regs_modified.back()[reg] = value; - m_last_set_operand_value = value; -} - -void ptx_thread_info::print_reg_thread(char *fname) { - FILE *fp = fopen(fname, "w"); - assert(fp != NULL); +//Using profiled information::check the TensorCoreMatrixArrangement.xls for details +unsigned thread_group_offset(int thread,unsigned wmma_type,unsigned wmma_layout,unsigned type,int stride){ + + unsigned offset; + unsigned load_a_row[8]={0,128,0,128,64,192,64,192}; + unsigned load_a_col[8]={0,8,0,8,4,12,4,12}; + unsigned load_b_row[8]={0,8,0,8,4,12,4,12}; + unsigned load_b_col[8]={0,128,0,128,64,192,64,192}; + unsigned load_c_float_row[8]={0,128,8,136,64,192,72,200}; + unsigned load_c_float_col[8]={0,8,128,136,4,12,132,140}; + unsigned load_c_half_row[8]={0,128,8,136,64,192,72,200}; + unsigned load_c_half_col[8]={0,8,128,136,4,12,132,140}; + unsigned thread_group = thread/4; + unsigned in_tg_index = thread%4; + + switch(wmma_type){ + case LOAD_A: + if(wmma_layout==ROW) + offset=load_a_row[thread_group]+16*in_tg_index; + else + offset=load_a_col[thread_group]+16*in_tg_index; + break; + + + case LOAD_B: + if(wmma_layout==ROW) + offset=load_b_row[thread_group]+16*in_tg_index; + else + offset=load_b_col[thread_group]+16*in_tg_index; + break; + + case LOAD_C: + case STORE_D: + if(type==F16_TYPE){ + if(wmma_layout==ROW) + offset=load_c_half_row[thread_group]+16*in_tg_index; + else + offset=load_c_half_col[thread_group]+in_tg_index; + } + else{ + if(wmma_layout==ROW) + offset=load_c_float_row[thread_group]; + else + offset=load_c_float_col[thread_group]; + + switch(in_tg_index){ + case 0: + break; + case 1: + if(wmma_layout==ROW) + offset+=16; + else + offset+=1; + break; + case 2: + if(wmma_layout==ROW) + offset+=2; + else + offset+=32; + break; + case 3: + if(wmma_layout==ROW) + offset+=18; + else + offset+=33; + break; + default: + abort(); + } + } + break; + + default: + abort(); + + } + offset = (offset/16)*stride+offset%16; + return offset; +} + +int acc_float_offset(int index,int wmma_layout,int stride){ + + int c_row_offset[]={0,1,32,33,4,5,36,37}; + int c_col_offset[]={0,16,2,18,64,80,66,82}; + int offset; + + + if(wmma_layout==ROW) + offset=c_row_offset[index]; + else if(wmma_layout==COL) + offset=c_col_offset[index]; + else{ + printf("wrong layout"); + abort(); + } + offset = (offset/16)*stride+offset%16; + return offset; +} + +void inst_not_implemented( const ptx_instruction * pI ) ; +ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread); + +void sign_extend( ptx_reg_t &data, unsigned src_size, const operand_info &dst ); + +void ptx_thread_info::set_reg( const symbol *reg, const ptx_reg_t &value ) +{ + assert( reg != NULL ); + if( reg->name() == "_" ) return; + assert( !m_regs.empty() ); + assert( reg->uid() > 0 ); + m_regs.back()[ reg ] = value; + if (m_enable_debug_trace ) + m_debug_trace_regs_modified.back()[ reg ] = value; + m_last_set_operand_value = value; +} + +void ptx_thread_info::print_reg_thread(char * fname) +{ + + FILE *fp= fopen(fname,"w"); + assert(fp!=NULL); int size = m_regs.size(); - - if (size > 0) { - reg_map_t reg = m_regs.back(); - - reg_map_t::const_iterator it; - for (it = reg.begin(); it != reg.end(); ++it) { - const std::string &name = it->first->name(); - const std::string &dec = it->first->decl_location(); - unsigned size = it->first->get_size_in_bytes(); - fprintf(fp, "%s %llu %s %d\n", name.c_str(), it->second, dec.c_str(), - size); - } - // m_regs.pop_back(); + + if(size>0) + { + reg_map_t reg = m_regs.back(); + + reg_map_t::const_iterator it; + for (it = reg.begin(); it != reg.end(); ++it) + { + const std::string &name = it->first->name(); + const std::string &dec= it->first->decl_location(); + unsigned size = it->first->get_size_in_bytes(); + fprintf(fp,"%s %llu %s %d\n", name.c_str(), it->second, dec.c_str(), size); + + } + //m_regs.pop_back(); } fclose(fp); -} - -void ptx_thread_info::resume_reg_thread(char *fname, symbol_table *symtab) { - FILE *fp2 = fopen(fname, "r"); - assert(fp2 != NULL); - // m_regs.push_back( reg_map_t() ); - char line[200]; - while (fgets(line, sizeof line, fp2) != NULL) { - symbol *reg; - char *pch; - pch = strtok(line, " "); - char *name = pch; - reg = symtab->lookup(name); - ptx_reg_t data; - pch = strtok(NULL, " "); - data = atoi(pch); - pch = strtok(NULL, " "); - pch = strtok(NULL, " "); - m_regs.back()[reg] = data; - } - fclose(fp2); -} -ptx_reg_t ptx_thread_info::get_reg(const symbol *reg) { - static bool unfound_register_warned = false; - assert(reg != NULL); - assert(!m_regs.empty()); - reg_map_t::iterator regs_iter = m_regs.back().find(reg); - if (regs_iter == m_regs.back().end()) { - assert(reg->type()->get_key().is_reg()); - const std::string &name = reg->name(); - unsigned call_uid = m_callstack.back().m_call_uid; - ptx_reg_t uninit_reg; - uninit_reg.u32 = 0x0; - set_reg(reg, uninit_reg); // give it a value since we are going to warn the - // user anyway - std::string file_loc = get_location(); - if (!unfound_register_warned) { - printf( - "GPGPU-Sim PTX: WARNING (%s) ** reading undefined register \'%s\' " - "(cuid:%u). Setting to 0X00000000. This is okay if you are " - "simulating the native ISA" - "\n", - file_loc.c_str(), name.c_str(), call_uid); - unfound_register_warned = true; - } - regs_iter = m_regs.back().find(reg); } - if (m_enable_debug_trace) - m_debug_trace_regs_read.back()[reg] = regs_iter->second; - return regs_iter->second; -} -ptx_reg_t ptx_thread_info::get_operand_value(const operand_info &op, - operand_info dstInfo, - unsigned opType, - ptx_thread_info *thread, - int derefFlag) { - ptx_reg_t result, tmp; - - if (op.get_double_operand_type() == 0) { - if (((opType != BB128_TYPE) && (opType != BB64_TYPE) && - (opType != FF64_TYPE)) || - (op.get_addr_space() != undefined_space)) { - if (op.is_reg()) { - result = get_reg(op.get_symbol()); - } else if (op.is_builtin()) { - result.u32 = get_builtin(op.get_int(), op.get_addr_offset()); - } else if (op.is_immediate_address()) { - result.u64 = op.get_addr_offset(); - } else if (op.is_memory_operand()) { - // a few options here... - const symbol *sym = op.get_symbol(); - const type_info *type = sym->type(); - const type_info_key &info = type->get_key(); - - if (info.is_reg()) { - const symbol *name = op.get_symbol(); - result.u64 = get_reg(name).u64 + op.get_addr_offset(); - } else if (info.is_param_kernel()) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if (info.is_param_local()) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if (info.is_global()) { - assert(op.get_addr_offset() == 0); - result.u64 = sym->get_address(); - } else if (info.is_local()) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if (info.is_const()) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if (op.is_shared()) { - result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); - } else if (op.is_sstarr()) { - result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); - } else { - const char *name = op.name().c_str(); - printf( - "GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory " - "operand type for %s\n", - name); - abort(); - } +void ptx_thread_info::resume_reg_thread(char * fname, symbol_table * symtab) +{ + + + FILE * fp2 = fopen(fname, "r"); + assert(fp2!=NULL); + //m_regs.push_back( reg_map_t() ); + char line [ 200 ]; + while ( fgets ( line, sizeof line, fp2 ) != NULL ) + { + symbol *reg; + char * pch; + pch = strtok (line," "); + char * name =pch; + reg= symtab->lookup(name); + ptx_reg_t data; + pch = strtok (NULL," "); + data = atoi(pch); + pch = strtok (NULL," "); + pch = strtok (NULL," "); + m_regs.back()[reg] = data; + } + fclose ( fp2 ); +} + + +ptx_reg_t ptx_thread_info::get_reg( const symbol *reg ) +{ + static bool unfound_register_warned = false; + assert( reg != NULL ); + assert( !m_regs.empty() ); + reg_map_t::iterator regs_iter = m_regs.back().find(reg); + if (regs_iter == m_regs.back().end()) { + assert( reg->type()->get_key().is_reg() ); + const std::string &name = reg->name(); + unsigned call_uid = m_callstack.back().m_call_uid; + ptx_reg_t uninit_reg; + uninit_reg.u32 = 0x0; + set_reg(reg, uninit_reg); // give it a value since we are going to warn the user anyway + std::string file_loc = get_location(); + if( !unfound_register_warned ) { + printf("GPGPU-Sim PTX: WARNING (%s) ** reading undefined register \'%s\' (cuid:%u). Setting to 0X00000000. This is okay if you are simulating the native ISA" + "\n", + file_loc.c_str(), name.c_str(), call_uid ); + unfound_register_warned = true; + } + regs_iter = m_regs.back().find(reg); + } + if (m_enable_debug_trace ) + m_debug_trace_regs_read.back()[ reg ] = regs_iter->second; + return regs_iter->second; +} + +ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_info dstInfo, unsigned opType, ptx_thread_info *thread, int derefFlag ) +{ + ptx_reg_t result, tmp; + + + if(op.get_double_operand_type() == 0) { + if(((opType != BB128_TYPE) && (opType != BB64_TYPE) && (opType != FF64_TYPE)) || (op.get_addr_space() != undefined_space)) { + if ( op.is_reg() ) { + result = get_reg( op.get_symbol() ); + } else if ( op.is_builtin()) { + result.u32 = get_builtin( op.get_int(), op.get_addr_offset() ); + } else if(op.is_immediate_address()){ + result.u64 = op.get_addr_offset(); + } else if ( op.is_memory_operand() ) { + // a few options here... + const symbol *sym = op.get_symbol(); + const type_info *type = sym->type(); + const type_info_key &info = type->get_key(); + + if ( info.is_reg() ) { + const symbol *name = op.get_symbol(); + result.u64 = get_reg(name).u64 + op.get_addr_offset(); + } else if ( info.is_param_kernel() ) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if ( info.is_param_local() ) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if ( info.is_global() ) { + assert( op.get_addr_offset() == 0 ); + result.u64 = sym->get_address(); + } else if ( info.is_local() ) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if ( info.is_const() ) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if ( op.is_shared() ) { + result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); + } else if ( op.is_sstarr() ) { + result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); + } else { + const char *name = op.name().c_str(); + printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory operand type for %s\n", name ); + abort(); + } - } else if (op.is_literal()) { - result = op.get_literal_value(); - } else if (op.is_label()) { - result.u64 = op.get_symbol()->get_address(); - } else if (op.is_shared()) { - result.u64 = op.get_symbol()->get_address(); - } else if (op.is_sstarr()) { - result.u64 = op.get_symbol()->get_address(); - } else if (op.is_const()) { - result.u64 = op.get_symbol()->get_address(); - } else if (op.is_global()) { - result.u64 = op.get_symbol()->get_address(); - } else if (op.is_local()) { - result.u64 = op.get_symbol()->get_address(); - } else if (op.is_function_address()) { - result.u64 = (size_t)op.get_symbol()->get_pc(); - } else if (op.is_param_kernel()) { - result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_literal() ) { + result = op.get_literal_value(); + } else if ( op.is_label() ) { + result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_shared() ) { + result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_sstarr() ) { + result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_const() ) { + result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_global() ) { + result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_local() ) { + result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_function_address() ) { + result.u64 = (size_t)op.get_symbol()->get_pc(); + } else if ( op.is_param_kernel()) { + result.u64 = op.get_symbol()->get_address(); + }else { + const char *name = op.name().c_str(); + const symbol *sym2 = op.get_symbol(); + const type_info *type2 = sym2->type(); + const type_info_key &info2 = type2->get_key(); + if ( info2.is_param_kernel() ) { + result.u64 = sym2->get_address()+ op.get_addr_offset(); + } + else{ + printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand type for %s\n", name ); + assert(0); + } + } + + if(op.get_operand_lohi() == 1) + result.u64 = result.u64 & 0xFFFF; + else if(op.get_operand_lohi() == 2) + result.u64 = (result.u64>>16) & 0xFFFF; + } else if (opType == BB128_TYPE) { + // b128 + result.u128.lowest = get_reg( op.vec_symbol(0) ).u32; + result.u128.low = get_reg( op.vec_symbol(1) ).u32; + result.u128.high = get_reg( op.vec_symbol(2) ).u32; + result.u128.highest = get_reg( op.vec_symbol(3) ).u32; } else { - const char *name = op.name().c_str(); - const symbol *sym2 = op.get_symbol(); - const type_info *type2 = sym2->type(); - const type_info_key &info2 = type2->get_key(); - if (info2.is_param_kernel()) { - result.u64 = sym2->get_address() + op.get_addr_offset(); - } else { - printf( - "GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand " - "type for %s\n", - name); - assert(0); - } + // bb64 or ff64 + result.bits.ls = get_reg( op.vec_symbol(0) ).u32; + result.bits.ms = get_reg( op.vec_symbol(1) ).u32; } - - if (op.get_operand_lohi() == 1) - result.u64 = result.u64 & 0xFFFF; - else if (op.get_operand_lohi() == 2) - result.u64 = (result.u64 >> 16) & 0xFFFF; - } else if (opType == BB128_TYPE) { - // b128 - result.u128.lowest = get_reg(op.vec_symbol(0)).u32; - result.u128.low = get_reg(op.vec_symbol(1)).u32; - result.u128.high = get_reg(op.vec_symbol(2)).u32; - result.u128.highest = get_reg(op.vec_symbol(3)).u32; - } else { - // bb64 or ff64 - result.bits.ls = get_reg(op.vec_symbol(0)).u32; - result.bits.ms = get_reg(op.vec_symbol(1)).u32; - } - } else if (op.get_double_operand_type() == 1) { - ptx_reg_t firstHalf, secondHalf; - firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; - secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; - if (op.get_operand_lohi() == 1) - secondHalf.u64 = secondHalf.u64 & 0xFFFF; - else if (op.get_operand_lohi() == 2) - secondHalf.u64 = (secondHalf.u64 >> 16) & 0xFFFF; - result.u64 = firstHalf.u64 + secondHalf.u64; - } else if (op.get_double_operand_type() == 2) { - // s[reg1 += reg2] - // reg1 is incremented after value is returned: the value returned is - // s[reg1] - ptx_reg_t firstHalf, secondHalf; - firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; - secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; - if (op.get_operand_lohi() == 1) - secondHalf.u64 = secondHalf.u64 & 0xFFFF; - else if (op.get_operand_lohi() == 2) - secondHalf.u64 = (secondHalf.u64 >> 16) & 0xFFFF; - result.u64 = firstHalf.u64; - firstHalf.u64 = firstHalf.u64 + secondHalf.u64; - set_reg(op.vec_symbol(0), firstHalf); - } else if (op.get_double_operand_type() == 3) { - // s[reg += immediate] - // reg is incremented after value is returned: the value returned is s[reg] - ptx_reg_t firstHalf; - firstHalf.u64 = get_reg(op.get_symbol()).u64; - result.u64 = firstHalf.u64; - firstHalf.u64 = firstHalf.u64 + op.get_addr_offset(); - set_reg(op.get_symbol(), firstHalf); - } - - ptx_reg_t finalResult; - memory_space *mem = NULL; - size_t size = 0; - int t = 0; - finalResult.u64 = 0; - - // complete other cases for reading from memory, such as reading from other - // const memory - if ((op.get_addr_space() == global_space) && (derefFlag)) { - // global memory - g[4], g[$r0] - mem = thread->get_global_memory(); - type_info_key::type_decode(opType, size, t); - mem->read(result.u32, size / 8, &finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = global_space; - - if (opType == S16_TYPE || opType == S32_TYPE) - sign_extend(finalResult, size, dstInfo); - } else if ((op.get_addr_space() == shared_space) && (derefFlag)) { - // shared memory - s[4], s[$r0] - mem = thread->m_shared_mem; - type_info_key::type_decode(opType, size, t); - mem->read(result.u32, size / 8, &finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = shared_space; - - if (opType == S16_TYPE || opType == S32_TYPE) - sign_extend(finalResult, size, dstInfo); - } else if ((op.get_addr_space() == const_space) && (derefFlag)) { - // const memory - ce0c1[4], ce0c1[$r0] - mem = thread->get_global_memory(); - type_info_key::type_decode(opType, size, t); - mem->read((result.u32 + op.get_const_mem_offset()), size / 8, - &finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = const_space; - if (opType == S16_TYPE || opType == S32_TYPE) - sign_extend(finalResult, size, dstInfo); - } else if ((op.get_addr_space() == local_space) && (derefFlag)) { - // local memory - l0[4], l0[$r0] - mem = thread->m_local_mem; - type_info_key::type_decode(opType, size, t); - mem->read(result.u32, size / 8, &finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = local_space; - if (opType == S16_TYPE || opType == S32_TYPE) - sign_extend(finalResult, size, dstInfo); - } else { - finalResult = result; - } - - if ((op.get_operand_neg() == true) && (derefFlag)) { - switch (opType) { + } else if (op.get_double_operand_type() == 1) { + ptx_reg_t firstHalf, secondHalf; + firstHalf.u64 = get_reg( op.vec_symbol(0) ).u64; + secondHalf.u64 = get_reg( op.vec_symbol(1) ).u64; + if(op.get_operand_lohi() == 1) + secondHalf.u64 = secondHalf.u64 & 0xFFFF; + else if(op.get_operand_lohi() == 2) + secondHalf.u64 = (secondHalf.u64>>16) & 0xFFFF; + result.u64 = firstHalf.u64 + secondHalf.u64; + } else if (op.get_double_operand_type() == 2) { + // s[reg1 += reg2] + // reg1 is incremented after value is returned: the value returned is s[reg1] + ptx_reg_t firstHalf, secondHalf; + firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; + secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; + if(op.get_operand_lohi() == 1) + secondHalf.u64 = secondHalf.u64 & 0xFFFF; + else if(op.get_operand_lohi() == 2) + secondHalf.u64 = (secondHalf.u64>>16) & 0xFFFF; + result.u64 = firstHalf.u64; + firstHalf.u64 = firstHalf.u64 + secondHalf.u64; + set_reg(op.vec_symbol(0),firstHalf); + } else if (op.get_double_operand_type() == 3) { + // s[reg += immediate] + // reg is incremented after value is returned: the value returned is s[reg] + ptx_reg_t firstHalf; + firstHalf.u64 = get_reg(op.get_symbol()).u64; + result.u64 = firstHalf.u64; + firstHalf.u64 = firstHalf.u64 + op.get_addr_offset(); + set_reg(op.get_symbol(),firstHalf); + } + + ptx_reg_t finalResult; + memory_space *mem = NULL; + size_t size=0; + int t=0; + finalResult.u64=0; + + //complete other cases for reading from memory, such as reading from other const memory + if((op.get_addr_space() == global_space)&&(derefFlag)) { + // global memory - g[4], g[$r0] + mem = thread->get_global_memory(); + type_info_key::type_decode(opType,size,t); + mem->read(result.u32,size/8,&finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = global_space; + + if( opType == S16_TYPE || opType == S32_TYPE ) + sign_extend(finalResult,size,dstInfo); + } else if((op.get_addr_space() == shared_space)&&(derefFlag)) { + // shared memory - s[4], s[$r0] + mem = thread->m_shared_mem; + type_info_key::type_decode(opType,size,t); + mem->read(result.u32,size/8,&finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = shared_space; + + if( opType == S16_TYPE || opType == S32_TYPE ) + sign_extend(finalResult,size,dstInfo); + } else if((op.get_addr_space() == const_space)&&(derefFlag)) { + // const memory - ce0c1[4], ce0c1[$r0] + mem = thread->get_global_memory(); + type_info_key::type_decode(opType,size,t); + mem->read((result.u32 + op.get_const_mem_offset()),size/8,&finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = const_space; + if( opType == S16_TYPE || opType == S32_TYPE ) + sign_extend(finalResult,size,dstInfo); + } else if((op.get_addr_space() == local_space)&&(derefFlag)) { + // local memory - l0[4], l0[$r0] + mem = thread->m_local_mem; + type_info_key::type_decode(opType,size,t); + mem->read(result.u32,size/8,&finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = local_space; + if( opType == S16_TYPE || opType == S32_TYPE ) + sign_extend(finalResult,size,dstInfo); + } else { + finalResult = result; + } + + if((op.get_operand_neg() == true)&&(derefFlag)) { + switch( opType ) { // Default to f32 for now, need to add support for others case S8_TYPE: case U8_TYPE: case B8_TYPE: - finalResult.s8 = -finalResult.s8; - break; + finalResult.s8 = -finalResult.s8; + break; case S16_TYPE: case U16_TYPE: case B16_TYPE: - finalResult.s16 = -finalResult.s16; - break; + finalResult.s16 = -finalResult.s16; + break; case S32_TYPE: case U32_TYPE: case B32_TYPE: - finalResult.s32 = -finalResult.s32; - break; + finalResult.s32 = -finalResult.s32; + break; case S64_TYPE: case U64_TYPE: case B64_TYPE: - finalResult.s64 = -finalResult.s64; - break; + finalResult.s64 = -finalResult.s64; + break; case F16_TYPE: - finalResult.f16 = -finalResult.f16; - break; + finalResult.f16 = -finalResult.f16; + break; case F32_TYPE: - finalResult.f32 = -finalResult.f32; - break; + finalResult.f32 = -finalResult.f32; + break; case F64_TYPE: case FF64_TYPE: - finalResult.f64 = -finalResult.f64; - break; + finalResult.f64 = -finalResult.f64; + break; default: - assert(0); - } - } - - return finalResult; -} + assert(0); + } -unsigned get_operand_nbits(const operand_info &op) { - if (op.is_reg()) { - const symbol *sym = op.get_symbol(); - const type_info *typ = sym->type(); - type_info_key t = typ->get_key(); - switch (t.scalar_type()) { - case PRED_TYPE: - return 1; - case B8_TYPE: - case S8_TYPE: - case U8_TYPE: - return 8; - case S16_TYPE: - case U16_TYPE: - case F16_TYPE: - case B16_TYPE: - return 16; - case S32_TYPE: - case U32_TYPE: - case F32_TYPE: - case B32_TYPE: - return 32; - case S64_TYPE: - case U64_TYPE: - case F64_TYPE: - case B64_TYPE: - return 64; + } + + return finalResult; + +} + +unsigned get_operand_nbits( const operand_info &op ) +{ + if ( op.is_reg() ) { + const symbol *sym = op.get_symbol(); + const type_info *typ = sym->type(); + type_info_key t = typ->get_key(); + switch( t.scalar_type() ) { + case PRED_TYPE: + return 1; + case B8_TYPE: case S8_TYPE: case U8_TYPE: + return 8; + case S16_TYPE: case U16_TYPE: case F16_TYPE: case B16_TYPE: + return 16; + case S32_TYPE: case U32_TYPE: case F32_TYPE: case B32_TYPE: + return 32; + case S64_TYPE: case U64_TYPE: case F64_TYPE: case B64_TYPE: + return 64; default: - printf("ERROR: unknown register type\n"); - fflush(stdout); - abort(); - } - } else { - printf( - "ERROR: Need to implement get_operand_nbits() for currently " - "unsupported operand_info type\n"); - fflush(stdout); - abort(); - } + printf("ERROR: unknown register type\n"); + fflush(stdout); + abort(); + } + } else { + printf("ERROR: Need to implement get_operand_nbits() for currently unsupported operand_info type\n"); + fflush(stdout); + abort(); + } - return 0; + return 0; } -void ptx_thread_info::get_vector_operand_values(const operand_info &op, - ptx_reg_t *ptx_regs, - unsigned num_elements) { - assert(op.is_vector()); - assert(num_elements <= 8); - - for (int idx = num_elements - 1; idx >= 0; --idx) { - const symbol *sym = NULL; - sym = op.vec_symbol(idx); - if (strcmp(sym->name().c_str(), "_") != 0) { - reg_map_t::iterator reg_iter = m_regs.back().find(sym); - assert(reg_iter != m_regs.back().end()); - ptx_regs[idx] = reg_iter->second; - } - } -} +void ptx_thread_info::get_vector_operand_values( const operand_info &op, ptx_reg_t* ptx_regs, unsigned num_elements ) +{ + assert( op.is_vector() ); + assert( num_elements <= 8 ); -void sign_extend(ptx_reg_t &data, unsigned src_size, const operand_info &dst) { - if (!dst.is_reg()) return; - unsigned dst_size = get_operand_nbits(dst); - if (src_size >= dst_size) return; - // src_size < dst_size - unsigned long long mask = 1; - mask <<= (src_size - 1); - if ((mask & data.u64) == 0) { - // no need to sign extend - return; - } - // need to sign extend - mask = 1; - mask <<= dst_size - src_size; - mask -= 1; - mask <<= src_size; - data.u64 |= mask; + for (int idx = num_elements - 1; idx >= 0; --idx) { + const symbol *sym = NULL; + sym = op.vec_symbol(idx); + if( strcmp(sym->name().c_str(),"_") != 0) { + reg_map_t::iterator reg_iter = m_regs.back().find(sym); + assert( reg_iter != m_regs.back().end() ); + ptx_regs[idx] = reg_iter->second; + } + } } -void ptx_thread_info::set_operand_value(const operand_info &dst, - const ptx_reg_t &data, unsigned type, - ptx_thread_info *thread, - const ptx_instruction *pI, int overflow, - int carry) { - thread->set_operand_value(dst, data, type, thread, pI); - - if (dst.get_double_operand_type() == -2) { - ptx_reg_t predValue; - - const symbol *sym = dst.vec_symbol(0); - predValue.u64 = (m_regs.back()[sym].u64) & ~(0x0C); - predValue.u64 |= ((overflow & 0x01) << 3); - predValue.u64 |= ((carry & 0x01) << 2); - - set_reg(sym, predValue); - } else if (dst.get_double_operand_type() == 0) { - // intentionally do nothing - } else { - printf("Unexpected double destination\n"); - assert(0); - } +void sign_extend( ptx_reg_t &data, unsigned src_size, const operand_info &dst ) +{ + if( !dst.is_reg() ) + return; + unsigned dst_size = get_operand_nbits( dst ); + if( src_size >= dst_size ) + return; + // src_size < dst_size + unsigned long long mask = 1; + mask <<= (src_size-1); + if( (mask & data.u64) == 0 ) { + // no need to sign extend + return; + } + // need to sign extend + mask = 1; + mask <<= dst_size-src_size; + mask -= 1; + mask <<= src_size; + data.u64 |= mask; } -void ptx_thread_info::set_operand_value(const operand_info &dst, - const ptx_reg_t &data, unsigned type, - ptx_thread_info *thread, - const ptx_instruction *pI) { - ptx_reg_t dstData; - memory_space *mem = NULL; - size_t size; - int t; - - type_info_key::type_decode(type, size, t); - - /*complete this section for other cases*/ - if (dst.get_addr_space() == undefined_space) { - ptx_reg_t setValue; - setValue.u64 = data.u64; - - // Double destination in set instruction ($p0|$p1) - second is negation of - // first - if (dst.get_double_operand_type() == -1) { - ptx_reg_t setValue2; - const symbol *name1 = dst.vec_symbol(0); - const symbol *name2 = dst.vec_symbol(1); - - if ((type == F16_TYPE) || (type == F32_TYPE) || (type == F64_TYPE) || - (type == FF64_TYPE)) { - setValue2.f32 = (setValue.u64 == 0) ? 1.0f : 0.0f; - } else { - setValue2.u32 = (setValue.u64 == 0) ? 0xFFFFFFFF : 0; - } +void ptx_thread_info::set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI, int overflow, int carry ) +{ + thread->set_operand_value( dst, data, type, thread, pI ); - set_reg(name1, setValue); - set_reg(name2, setValue2); + if (dst.get_double_operand_type() == -2) + { + ptx_reg_t predValue; + + const symbol *sym = dst.vec_symbol(0); + predValue.u64 = (m_regs.back()[ sym ].u64) & ~(0x0C); + predValue.u64 |= ((overflow & 0x01)<<3); + predValue.u64 |= ((carry & 0x01)<<2); + + set_reg(sym,predValue); } - - // Double destination in cvt,shr,mul,etc. instruction ($p0|$r4) - second - // register operand receives data, first predicate operand - // is set as $p0=($r4!=0) - // Also for Double destination in set instruction ($p0/$r1) - else if ((dst.get_double_operand_type() == -2) || - (dst.get_double_operand_type() == -3)) { - ptx_reg_t predValue; - const symbol *predName = dst.vec_symbol(0); - const symbol *regName = dst.vec_symbol(1); - predValue.u64 = 0; - - switch (type) { - case S8_TYPE: - if ((setValue.s8 & 0x7F) == 0) predValue.u64 |= 1; - break; - case S16_TYPE: - if ((setValue.s16 & 0x7FFF) == 0) predValue.u64 |= 1; - break; - case S32_TYPE: - if ((setValue.s32 & 0x7FFFFFFF) == 0) predValue.u64 |= 1; - break; - case S64_TYPE: - if ((setValue.s64 & 0x7FFFFFFFFFFFFFFF) == 0) predValue.u64 |= 1; - break; - case U8_TYPE: - case B8_TYPE: - if (setValue.u8 == 0) predValue.u64 |= 1; - break; - case U16_TYPE: - case B16_TYPE: - if (setValue.u16 == 0) predValue.u64 |= 1; - break; - case U32_TYPE: - case B32_TYPE: - if (setValue.u32 == 0) predValue.u64 |= 1; - break; - case U64_TYPE: - case B64_TYPE: - if (setValue.u64 == 0) predValue.u64 |= 1; - break; - case F16_TYPE: - if (setValue.f16 == 0) predValue.u64 |= 1; - break; - case F32_TYPE: - if (setValue.f32 == 0) predValue.u64 |= 1; - break; - case F64_TYPE: - case FF64_TYPE: - if (setValue.f64 == 0) predValue.u64 |= 1; - break; - default: - assert(0); - break; - } - - if ((type == S8_TYPE) || (type == S16_TYPE) || (type == S32_TYPE) || - (type == S64_TYPE) || (type == U8_TYPE) || (type == U16_TYPE) || - (type == U32_TYPE) || (type == U64_TYPE) || (type == B8_TYPE) || - (type == B16_TYPE) || (type == B32_TYPE) || (type == B64_TYPE)) { - if ((setValue.u32 & (1 << (size - 1))) != 0) predValue.u64 |= 1 << 1; - } - if (type == F32_TYPE) { - if (setValue.f32 < 0) predValue.u64 |= 1 << 1; - } - - if (dst.get_operand_lohi() == 1) { - setValue.u64 = - ((m_regs.back()[regName].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); - } else if (dst.get_operand_lohi() == 2) { - setValue.u64 = ((m_regs.back()[regName].u64) & (~(0xFFFF0000))) + - ((data.u64 << 16) & 0xFFFF0000); - } - - set_reg(predName, predValue); - set_reg(regName, setValue); - } else if (type == BB128_TYPE) { - // b128 stuff here. - ptx_reg_t setValue2, setValue3, setValue4; - setValue.u64 = 0; - setValue2.u64 = 0; - setValue3.u64 = 0; - setValue4.u64 = 0; - setValue.u32 = data.u128.lowest; - setValue2.u32 = data.u128.low; - setValue3.u32 = data.u128.high; - setValue4.u32 = data.u128.highest; - - const symbol *name1, *name2, *name3, *name4 = NULL; - - name1 = dst.vec_symbol(0); - name2 = dst.vec_symbol(1); - name3 = dst.vec_symbol(2); - name4 = dst.vec_symbol(3); - - set_reg(name1, setValue); - set_reg(name2, setValue2); - set_reg(name3, setValue3); - set_reg(name4, setValue4); - } else if (type == BB64_TYPE || type == FF64_TYPE) { - // ptxplus version of storing 64 bit values to registers stores to two - // adjacent registers - ptx_reg_t setValue2; - setValue.u32 = 0; - setValue2.u32 = 0; - - setValue.u32 = data.bits.ls; - setValue2.u32 = data.bits.ms; - - const symbol *name1, *name2 = NULL; - - name1 = dst.vec_symbol(0); - name2 = dst.vec_symbol(1); - - set_reg(name1, setValue); - set_reg(name2, setValue2); - } else { - if (dst.get_operand_lohi() == 1) { - setValue.u64 = ((m_regs.back()[dst.get_symbol()].u64) & (~(0xFFFF))) + - (data.u64 & 0xFFFF); - } else if (dst.get_operand_lohi() == 2) { - setValue.u64 = - ((m_regs.back()[dst.get_symbol()].u64) & (~(0xFFFF0000))) + - ((data.u64 << 16) & 0xFFFF0000); - } - set_reg(dst.get_symbol(), setValue); + else if (dst.get_double_operand_type() == 0) + { + //intentionally do nothing + } + else + { + printf("Unexpected double destination\n"); + assert(0); } - } - - // global memory - g[4], g[$r0] - else if (dst.get_addr_space() == global_space) { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->get_global_memory(); - type_info_key::type_decode(type, size, t); - - mem->write(dstData.u32, size / 8, &data.u128, thread, pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = global_space; - } - - // shared memory - s[4], s[$r0] - else if (dst.get_addr_space() == shared_space) { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->m_shared_mem; - type_info_key::type_decode(type, size, t); - - mem->write(dstData.u32, size / 8, &data.u128, thread, pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = shared_space; - } - - // local memory - l0[4], l0[$r0] - else if (dst.get_addr_space() == local_space) { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->m_local_mem; - type_info_key::type_decode(type, size, t); - - mem->write(dstData.u32, size / 8, &data.u128, thread, pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = local_space; - } - else { - printf("Destination stores to unknown location."); - assert(0); - } } -void ptx_thread_info::set_vector_operand_values(const operand_info &dst, - const ptx_reg_t &data1, - const ptx_reg_t &data2, - const ptx_reg_t &data3, - const ptx_reg_t &data4) { - unsigned num_elements = dst.get_vect_nelem(); - if (num_elements > 0) { - set_reg(dst.vec_symbol(0), data1); - if (num_elements > 1) { - set_reg(dst.vec_symbol(1), data2); - if (num_elements > 2) { - set_reg(dst.vec_symbol(2), data3); - if (num_elements > 3) { - set_reg(dst.vec_symbol(3), data4); - } - } - } - } +void ptx_thread_info::set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI ) +{ + ptx_reg_t dstData; + memory_space *mem = NULL; + size_t size; + int t; - m_last_set_operand_value = data1; -} -void ptx_thread_info::set_wmma_vector_operand_values( - const operand_info &dst, const ptx_reg_t &data1, const ptx_reg_t &data2, - const ptx_reg_t &data3, const ptx_reg_t &data4, const ptx_reg_t &data5, - const ptx_reg_t &data6, const ptx_reg_t &data7, const ptx_reg_t &data8) { - unsigned num_elements = dst.get_vect_nelem(); - if (num_elements == 8) { - set_reg(dst.vec_symbol(0), data1); - set_reg(dst.vec_symbol(1), data2); - set_reg(dst.vec_symbol(2), data3); - set_reg(dst.vec_symbol(3), data4); - set_reg(dst.vec_symbol(4), data5); - set_reg(dst.vec_symbol(5), data6); - set_reg(dst.vec_symbol(6), data7); - set_reg(dst.vec_symbol(7), data8); - } else { - printf("error:set_wmma_vector_operands"); - } + type_info_key::type_decode(type,size,t); - m_last_set_operand_value = data8; -} + /*complete this section for other cases*/ + if(dst.get_addr_space() == undefined_space) + { + ptx_reg_t setValue; + setValue.u64 = data.u64; -#define my_abs(a) (((a) < 0) ? (-a) : (a)) + // Double destination in set instruction ($p0|$p1) - second is negation of first + if (dst.get_double_operand_type() == -1) + { + ptx_reg_t setValue2; + const symbol *name1 = dst.vec_symbol(0); + const symbol *name2 = dst.vec_symbol(1); -#define MY_MAX_I(a, b) (a > b) ? a : b -#define MY_MAX_F(a, b) isNaN(a) ? b : isNaN(b) ? a : (a > b) ? a : b + if ( (type==F16_TYPE)||(type==F32_TYPE)||(type==F64_TYPE)||(type==FF64_TYPE) ) { + setValue2.f32 = (setValue.u64==0)?1.0f:0.0f; + } else { + setValue2.u32 = (setValue.u64==0)?0xFFFFFFFF:0; + } -#define MY_MIN_I(a, b) (a < b) ? a : b -#define MY_MIN_F(a, b) isNaN(a) ? b : isNaN(b) ? a : (a < b) ? a : b + set_reg(name1,setValue); + set_reg(name2,setValue2); + } -#define MY_INC_I(a, b) (a >= b) ? 0 : a + 1 -#define MY_DEC_I(a, b) ((a == 0) || (a > b)) ? b : a - 1 + // Double destination in cvt,shr,mul,etc. instruction ($p0|$r4) - second register operand receives data, first predicate operand + // is set as $p0=($r4!=0) + // Also for Double destination in set instruction ($p0/$r1) + else if ((dst.get_double_operand_type() == -2)||(dst.get_double_operand_type() == -3)) + { + ptx_reg_t predValue; + const symbol *predName = dst.vec_symbol(0); + const symbol *regName = dst.vec_symbol(1); + predValue.u64 = 0; + + switch ( type ) { + case S8_TYPE: + if((setValue.s8 & 0x7F) == 0) + predValue.u64 |= 1; + break; + case S16_TYPE: + if((setValue.s16 & 0x7FFF) == 0) + predValue.u64 |= 1; + break; + case S32_TYPE: + if((setValue.s32 & 0x7FFFFFFF) == 0) + predValue.u64 |= 1; + break; + case S64_TYPE: + if((setValue.s64 & 0x7FFFFFFFFFFFFFFF) == 0) + predValue.u64 |= 1; + break; + case U8_TYPE: + case B8_TYPE: + if(setValue.u8 == 0) + predValue.u64 |= 1; + break; + case U16_TYPE: + case B16_TYPE: + if(setValue.u16 == 0) + predValue.u64 |= 1; + break; + case U32_TYPE: + case B32_TYPE: + if(setValue.u32 == 0) + predValue.u64 |= 1; + break; + case U64_TYPE: + case B64_TYPE: + if(setValue.u64 == 0) + predValue.u64 |= 1; + break; + case F16_TYPE: + if(setValue.f16 == 0) + predValue.u64 |= 1; + break; + case F32_TYPE: + if(setValue.f32 == 0) + predValue.u64 |= 1; + break; + case F64_TYPE: + case FF64_TYPE: + if(setValue.f64 == 0) + predValue.u64 |= 1; + break; + default: assert(0); break; + } -#define MY_CAS_I(a, b, c) (a == b) ? c : a -#define MY_EXCH(a, b) b + if ( (type==S8_TYPE)||(type==S16_TYPE)||(type==S32_TYPE)||(type==S64_TYPE)|| + (type==U8_TYPE)||(type==U16_TYPE)||(type==U32_TYPE)||(type==U64_TYPE)|| + (type==B8_TYPE)||(type==B16_TYPE)||(type==B32_TYPE)||(type==B64_TYPE)) { + if((setValue.u32 & (1<<(size-1))) != 0) + predValue.u64 |= 1<<1; + } + if ( type==F32_TYPE ) { + if(setValue.f32 < 0) + predValue.u64 |= 1<<1; + } -void abs_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); + if(dst.get_operand_lohi() == 1) + { + setValue.u64 = ((m_regs.back()[ regName ].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); + } + else if(dst.get_operand_lohi() == 2) + { + setValue.u64 = ((m_regs.back()[ regName ].u64) & (~(0xFFFF0000))) + ((data.u64<<16) & 0xFFFF0000); + } - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + set_reg(predName,predValue); + set_reg(regName,setValue); + } + else if (type == BB128_TYPE) + { + //b128 stuff here. + ptx_reg_t setValue2, setValue3, setValue4; + setValue.u64 = 0; + setValue2.u64 = 0; + setValue3.u64 = 0; + setValue4.u64 = 0; + setValue.u32 = data.u128.lowest; + setValue2.u32 = data.u128.low; + setValue3.u32 = data.u128.high; + setValue4.u32 = data.u128.highest; + + const symbol *name1, *name2, *name3, *name4 = NULL; + + name1 = dst.vec_symbol(0); + name2 = dst.vec_symbol(1); + name3 = dst.vec_symbol(2); + name4 = dst.vec_symbol(3); + + set_reg(name1,setValue); + set_reg(name2,setValue2); + set_reg(name3,setValue3); + set_reg(name4,setValue4); + } + else if (type == BB64_TYPE || type == FF64_TYPE) + { + //ptxplus version of storing 64 bit values to registers stores to two adjacent registers + ptx_reg_t setValue2; + setValue.u32 = 0; + setValue2.u32 = 0; - switch (i_type) { - case S16_TYPE: - d.s16 = my_abs(a.s16); - break; - case S32_TYPE: - d.s32 = my_abs(a.s32); - break; - case S64_TYPE: - d.s64 = my_abs(a.s64); - break; - case U16_TYPE: - d.s16 = my_abs(a.u16); - break; - case U32_TYPE: - d.s32 = my_abs(a.u32); - break; - case U64_TYPE: - d.s64 = my_abs(a.u64); - break; - case F32_TYPE: - d.f32 = my_abs(a.f32); - break; - case F64_TYPE: - case FF64_TYPE: - d.f64 = my_abs(a.f64); - break; - default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } + setValue.u32 = data.bits.ls; + setValue2.u32 = data.bits.ms; - thread->set_operand_value(dst, d, i_type, thread, pI); -} + const symbol *name1, *name2 = NULL; -void addp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - // PTXPlus add instruction with carry (carry is kept in a predicate) register - ptx_reg_t src1_data, src2_data, src3_data, data; - int overflow = 0; - int carry = 0; - - const operand_info &dst = - pI->dst(); // get operand info of sources and destination - const operand_info &src1 = - pI->src1(); // use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - - unsigned rounding_mode = pI->rounding_mode(); - int orig_rm = fegetround(); - switch (rounding_mode) { - case RN_OPTION: - break; - case RZ_OPTION: - fesetround(FE_TOWARDZERO); - break; - default: - assert(0); - break; - } + name1 = dst.vec_symbol(0); + name2 = dst.vec_symbol(1); - // performs addition. Sets carry and overflow if needed. - // src3_data.pred&0x4 is the carry flag - switch (i_type) { - case S8_TYPE: - data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF) + - (src3_data.pred & 0x4); - if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) == 0) { - overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; + set_reg(name1,setValue); + set_reg(name2,setValue2); } - carry = (data.u64 & 0x000000100) >> 8; - break; - case S16_TYPE: - data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF) + - (src3_data.pred & 0x4); - if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) == 0) { - overflow = - ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; - } - carry = (data.u64 & 0x000010000) >> 16; - break; - case S32_TYPE: - data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF) + - (src3_data.pred & 0x4); - if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) == 0) { - overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 - ? 0 - : 1; + else + { + if(dst.get_operand_lohi() == 1) + { + setValue.u64 = ((m_regs.back()[ dst.get_symbol() ].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); + } + else if(dst.get_operand_lohi() == 2) + { + setValue.u64 = ((m_regs.back()[ dst.get_symbol() ].u64) & (~(0xFFFF0000))) + ((data.u64<<16) & 0xFFFF0000); + } + set_reg(dst.get_symbol(),setValue); } - carry = (data.u64 & 0x100000000) >> 32; + } + + // global memory - g[4], g[$r0] + else if(dst.get_addr_space() == global_space) + { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->get_global_memory(); + type_info_key::type_decode(type,size,t); + + mem->write(dstData.u32,size/8,&data.u128,thread,pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = global_space; + } + + // shared memory - s[4], s[$r0] + else if(dst.get_addr_space() == shared_space) + { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->m_shared_mem; + type_info_key::type_decode(type,size,t); + + mem->write(dstData.u32,size/8,&data.u128,thread,pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = shared_space; + } + + // local memory - l0[4], l0[$r0] + else if(dst.get_addr_space() == local_space) + { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->m_local_mem; + type_info_key::type_decode(type,size,t); + + mem->write(dstData.u32,size/8,&data.u128,thread,pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = local_space; + } + + else + { + printf("Destination stores to unknown location."); + assert(0); + } + + +} + +void ptx_thread_info::set_vector_operand_values( const operand_info &dst, + const ptx_reg_t &data1, + const ptx_reg_t &data2, + const ptx_reg_t &data3, + const ptx_reg_t &data4 ) +{ + unsigned num_elements = dst.get_vect_nelem(); + if (num_elements > 0) { + set_reg(dst.vec_symbol(0), data1); + if (num_elements > 1) { + set_reg(dst.vec_symbol(1), data2); + if (num_elements > 2) { + set_reg(dst.vec_symbol(2), data3); + if (num_elements > 3) { + set_reg(dst.vec_symbol(3), data4); + } + } + } + } + + m_last_set_operand_value = data1; +} +void ptx_thread_info::set_wmma_vector_operand_values( const operand_info &dst, + const ptx_reg_t &data1, + const ptx_reg_t &data2, + const ptx_reg_t &data3, + const ptx_reg_t &data4, + const ptx_reg_t &data5, + const ptx_reg_t &data6, + const ptx_reg_t &data7, + const ptx_reg_t &data8 ) +{ + unsigned num_elements = dst.get_vect_nelem(); + if (num_elements == 8) { + set_reg(dst.vec_symbol(0), data1); + set_reg(dst.vec_symbol(1), data2); + set_reg(dst.vec_symbol(2), data3); + set_reg(dst.vec_symbol(3), data4); + set_reg(dst.vec_symbol(4), data5); + set_reg(dst.vec_symbol(5), data6); + set_reg(dst.vec_symbol(6), data7); + set_reg(dst.vec_symbol(7), data8); + } + else{ + printf("error:set_wmma_vector_operands"); + } + + m_last_set_operand_value = data8; +} + +#define my_abs(a) (((a)<0)?(-a):(a)) + +#define MY_MAX_I(a,b) (a > b) ? a : b +#define MY_MAX_F(a,b) isNaN(a) ? b : isNaN(b) ? a : (a > b) ? a : b + +#define MY_MIN_I(a,b) (a < b) ? a : b +#define MY_MIN_F(a,b) isNaN(a) ? b : isNaN(b) ? a : (a < b) ? a : b + +#define MY_INC_I(a,b) (a >= b) ? 0 : a+1 +#define MY_DEC_I(a,b) ((a == 0) || (a > b)) ? b : a-1 + +#define MY_CAS_I(a,b,c) (a == b) ? c : a + +#define MY_EXCH(a,b) b + +void abs_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + + switch ( i_type ) { + case S16_TYPE: d.s16 = my_abs(a.s16); break; + case S32_TYPE: d.s32 = my_abs(a.s32); break; + case S64_TYPE: d.s64 = my_abs(a.s64); break; + case U16_TYPE: d.s16 = my_abs(a.u16); break; + case U32_TYPE: d.s32 = my_abs(a.u32); break; + case U64_TYPE: d.s64 = my_abs(a.u64); break; + case F32_TYPE: d.f32 = my_abs(a.f32); break; + case F64_TYPE: case FF64_TYPE: d.f64 = my_abs(a.f64); break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); break; - case S64_TYPE: + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +void addp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + //PTXPlus add instruction with carry (carry is kept in a predicate) register + ptx_reg_t src1_data, src2_data, src3_data, data; + int overflow = 0; + int carry = 0; + + const operand_info &dst = pI->dst(); //get operand info of sources and destination + const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + + unsigned rounding_mode = pI->rounding_mode(); + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + + //performs addition. Sets carry and overflow if needed. + //src3_data.pred&0x4 is the carry flag + switch ( i_type ) { + case S8_TYPE: + data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF) + (src3_data.pred & 0x4); + if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) == 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } + carry = (data.u64 & 0x000000100)>>8; + break; + case S16_TYPE: + data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF) + (src3_data.pred & 0x4); + if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) == 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } + carry = (data.u64 & 0x000010000)>>16; + break; + case S32_TYPE: + data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF) + (src3_data.pred & 0x4); + if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) == 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } + carry = (data.u64 & 0x100000000)>>32; + break; + case S64_TYPE: data.s64 = src1_data.s64 + src2_data.s64 + (src3_data.pred & 0x4); break; - case U8_TYPE: - data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF) + - (src3_data.pred & 0x4); - carry = (data.u64 & 0x100) >> 8; + case U8_TYPE: + data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF) + (src3_data.pred & 0x4); + carry = (data.u64 & 0x100)>>8; break; - case U16_TYPE: - data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF) + - (src3_data.pred & 0x4); - carry = (data.u64 & 0x10000) >> 16; + case U16_TYPE: + data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF) + (src3_data.pred & 0x4); + carry = (data.u64 & 0x10000)>>16; break; - case U32_TYPE: - data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF) + - (src3_data.pred & 0x4); - carry = (data.u64 & 0x100000000) >> 32; + case U32_TYPE: + data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF) + (src3_data.pred & 0x4); + carry = (data.u64 & 0x100000000)>>32; break; - case U64_TYPE: + case U64_TYPE: data.s64 = src1_data.s64 + src2_data.s64 + (src3_data.pred & 0x4); break; - case F16_TYPE: - data.f16 = src1_data.f16 + src2_data.f16; - break; // assert(0); break; - case F32_TYPE: - data.f32 = src1_data.f32 + src2_data.f32; - break; - case F64_TYPE: - case FF64_TYPE: - data.f64 = src1_data.f64 + src2_data.f64; - break; - default: - assert(0); - break; - } - fesetround(orig_rm); - - thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); -} - -void add_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; - int overflow = 0; - int carry = 0; - - const operand_info &dst = - pI->dst(); // get operand info of sources and destination - const operand_info &src1 = - pI->src1(); // use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - - unsigned rounding_mode = pI->rounding_mode(); - int orig_rm = fegetround(); - switch (rounding_mode) { - case RN_OPTION: - break; - case RZ_OPTION: - fesetround(FE_TOWARDZERO); - break; - default: - assert(0); - break; - } - - // performs addition. Sets carry and overflow if needed. - switch (i_type) { - case S8_TYPE: + case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break; + case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; + case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; + default: assert(0); break; + } + fesetround( orig_rm ); + + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry ); +} + +void add_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + int overflow = 0; + int carry = 0; + + const operand_info &dst = pI->dst(); //get operand info of sources and destination + const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + unsigned rounding_mode = pI->rounding_mode(); + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + + //performs addition. Sets carry and overflow if needed. + switch ( i_type ) { + case S8_TYPE: data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF); - if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) == 0) { - overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; - } - carry = (data.u64 & 0x000000100) >> 8; + if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) == 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } + carry = (data.u64 & 0x000000100)>>8; break; - case S16_TYPE: + case S16_TYPE: data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF); - if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) == 0) { - overflow = - ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; - } - carry = (data.u64 & 0x000010000) >> 16; + if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) == 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } + carry = (data.u64 & 0x000010000)>>16; break; - case S32_TYPE: + case S32_TYPE: data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF); - if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) == 0) { - overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 - ? 0 - : 1; - } - carry = (data.u64 & 0x100000000) >> 32; + if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) == 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } + carry = (data.u64 & 0x100000000)>>32; break; - case S64_TYPE: + case S64_TYPE: data.s64 = src1_data.s64 + src2_data.s64; break; - case U8_TYPE: + case U8_TYPE: data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF); - carry = (data.u64 & 0x100) >> 8; + carry = (data.u64 & 0x100)>>8; break; - case U16_TYPE: + case U16_TYPE: data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF); - carry = (data.u64 & 0x10000) >> 16; + carry = (data.u64 & 0x10000)>>16; break; - case U32_TYPE: + case U32_TYPE: data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF); - carry = (data.u64 & 0x100000000) >> 32; + carry = (data.u64 & 0x100000000)>>32; break; - case U64_TYPE: + case U64_TYPE: data.u64 = src1_data.u64 + src2_data.u64; break; - case F16_TYPE: - data.f16 = src1_data.f16 + src2_data.f16; - break; // assert(0); break; - case F32_TYPE: - data.f32 = src1_data.f32 + src2_data.f32; - break; - case F64_TYPE: - case FF64_TYPE: - data.f64 = src1_data.f64 + src2_data.f64; - break; - default: - assert(0); - break; - } - fesetround(orig_rm); - - thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); -} - -void addc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} - -void and_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break; + case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; + case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; + default: assert(0); break; + } + fesetround( orig_rm ); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - - // the way ptxplus handles predicates: 1 = false and 0 = true - if (i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) & ~(src2_data.pred)); - else - data.u64 = src1_data.u64 & src2_data.u64; - - thread->set_operand_value(dst, data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry ); } -void andn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; +void addc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void and_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - switch (i_type) { - case B16_TYPE: - src2_data.u16 = ~src2_data.u16; - break; - case B32_TYPE: - src2_data.u32 = ~src2_data.u32; - break; - case B64_TYPE: - src2_data.u64 = ~src2_data.u64; - break; - default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - data.u64 = src1_data.u64 & src2_data.u64; - thread->set_operand_value(dst, data, i_type, thread, pI); -} + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) & ~(src2_data.pred)); + else + data.u64 = src1_data.u64 & src2_data.u64; -void bar_callback(const inst_t *inst, ptx_thread_info *thread) { - unsigned ctaid = thread->get_cta_uid(); - unsigned barid = inst->bar_id; - unsigned value = thread->get_reduction_value(ctaid, barid); - const ptx_instruction *pI = dynamic_cast(inst); - const operand_info &dst = pI->dst(); - ptx_reg_t data; - data.u32 = value; - thread->set_operand_value(dst, value, U32_TYPE, thread, pI); + thread->set_operand_value(dst,data, i_type, thread, pI); } -void atom_callback(const inst_t *inst, ptx_thread_info *thread) { - const ptx_instruction *pI = dynamic_cast(inst); - - // "Decode" the output type - unsigned to_type = pI->get_type(); - size_t size; - int t; - type_info_key::type_decode(to_type, size, t); - - // Set up operand variables - ptx_reg_t data; // d - ptx_reg_t src1_data; // a - ptx_reg_t src2_data; // b - ptx_reg_t op_result; // temp variable to hold operation result - - bool data_ready = false; - - // Get operand info of sources and destination - const operand_info &dst = pI->dst(); // d - const operand_info &src1 = pI->src1(); // a - const operand_info &src2 = pI->src2(); // b - - // Get operand values - src1_data = thread->get_operand_value(src1, src1, to_type, thread, 1); // a - if (dst.get_symbol()->type()) { - src2_data = thread->get_operand_value(src2, dst, to_type, thread, 1); // b - } else { - // This is the case whent he first argument (dest) is '_' - src2_data = thread->get_operand_value(src2, src1, to_type, thread, 1); // b - } +void andn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; - // Check state space - addr_t effective_address = src1_data.u64; - memory_space_t space = pI->get_space(); - if (space == undefined_space) { - // generic space - determine space via address - if (whichspace(effective_address) == global_space) { - effective_address = generic_to_global(effective_address); - space = global_space; - } else if (whichspace(effective_address) == shared_space) { - unsigned smid = thread->get_hw_sid(); - effective_address = generic_to_shared(smid, effective_address); - space = shared_space; - } else { - abort(); - } - } - assert(space == global_space || space == shared_space); - - memory_space *mem = NULL; - if (space == global_space) - mem = thread->get_global_memory(); - else if (space == shared_space) - mem = thread->m_shared_mem; - else - abort(); - - // Copy value pointed to in operand 'a' into register 'd' - // (i.e. copy src1_data to dst) - mem->read(effective_address, size / 8, &data.s64); - if (dst.get_symbol()->type()) { - thread->set_operand_value(dst, data, to_type, thread, - pI); // Write value into register 'd' - } + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - // Get the atomic operation to be performed - unsigned m_atomic_spec = pI->get_atomic(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch (m_atomic_spec) { - // AND - case ATOMIC_AND: { - switch (to_type) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 & src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 & src2_data.s32; - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch (%x) with instruction\natom.AND " - "only accepts b32\n", - to_type); - assert(0); - break; + switch ( i_type ) { + case B16_TYPE: src2_data.u16 = ~src2_data.u16; break; + case B32_TYPE: src2_data.u32 = ~src2_data.u32; break; + case B64_TYPE: src2_data.u64 = ~src2_data.u64; break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + data.u64 = src1_data.u64 & src2_data.u64; + + thread->set_operand_value(dst,data, i_type, thread, pI); +} + +void bar_callback( const inst_t* inst, ptx_thread_info* thread) +{ + unsigned ctaid = thread->get_cta_uid(); + unsigned barid = inst->bar_id; + unsigned value = thread->get_reduction_value(ctaid,barid); + const ptx_instruction *pI = dynamic_cast(inst); + const operand_info &dst = pI->dst(); + ptx_reg_t data; + data.u32 = value; + thread->set_operand_value(dst,value, U32_TYPE, thread, pI); +} + +void atom_callback( const inst_t* inst, ptx_thread_info* thread) +{ + const ptx_instruction *pI = dynamic_cast(inst); + + // "Decode" the output type + unsigned to_type = pI->get_type(); + size_t size; + int t; + type_info_key::type_decode(to_type, size, t); + + // Set up operand variables + ptx_reg_t data; // d + ptx_reg_t src1_data; // a + ptx_reg_t src2_data; // b + ptx_reg_t op_result; // temp variable to hold operation result + + bool data_ready = false; + + // Get operand info of sources and destination + const operand_info &dst = pI->dst(); // d + const operand_info &src1 = pI->src1(); // a + const operand_info &src2 = pI->src2(); // b + + // Get operand values + src1_data = thread->get_operand_value(src1, src1, to_type, thread, 1); // a + if (dst.get_symbol()->type()){ + src2_data = thread->get_operand_value(src2, dst, to_type, thread, 1); // b + } else { + //This is the case whent he first argument (dest) is '_' + src2_data = thread->get_operand_value(src2, src1, to_type, thread, 1); // b + } + + // Check state space + addr_t effective_address = src1_data.u64; + memory_space_t space = pI->get_space(); + if (space == undefined_space) { + // generic space - determine space via address + if( whichspace(effective_address) == global_space ) { + effective_address = generic_to_global(effective_address); + space = global_space; + } else if( whichspace(effective_address) == shared_space ) { + unsigned smid = thread->get_hw_sid(); + effective_address = generic_to_shared(smid,effective_address); + space = shared_space; + } else { + abort(); } + } + assert( space == global_space || space == shared_space ); + + memory_space *mem = NULL; + if(space == global_space) + mem = thread->get_global_memory(); + else if(space == shared_space) + mem = thread->m_shared_mem; + else + abort(); + + // Copy value pointed to in operand 'a' into register 'd' + // (i.e. copy src1_data to dst) + mem->read(effective_address,size/8,&data.s64); + if (dst.get_symbol()->type()){ + thread->set_operand_value(dst, data, to_type, thread, pI); // Write value into register 'd' + } + + // Get the atomic operation to be performed + unsigned m_atomic_spec = pI->get_atomic(); + + switch ( m_atomic_spec ) { + // AND + case ATOMIC_AND: + { + + switch ( to_type ) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 & src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 & src2_data.s32; + data_ready = true; + break; + default: + printf("Execution error: type mismatch (%x) with instruction\natom.AND only accepts b32\n", to_type); + assert(0); + break; + } - break; - } - // OR - case ATOMIC_OR: { - switch (to_type) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 | src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 | src2_data.s32; - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch (%x) with instruction\natom.OR " - "only accepts b32\n", - to_type); - assert(0); - break; + break; } + // OR + case ATOMIC_OR: + { + + switch ( to_type ) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 | src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 | src2_data.s32; + data_ready = true; + break; + default: + printf("Execution error: type mismatch (%x) with instruction\natom.OR only accepts b32\n", to_type); + assert(0); + break; + } - break; - } - // XOR - case ATOMIC_XOR: { - switch (to_type) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 ^ src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 ^ src2_data.s32; - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch (%x) with instruction\natom.XOR " - "only accepts b32\n", - to_type); - assert(0); - break; - } - - break; - } - // CAS - case ATOMIC_CAS: { - ptx_reg_t src3_data; - const operand_info &src3 = pI->src3(); - src3_data = thread->get_operand_value(src3, dst, to_type, thread, 1); - - switch (to_type) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = MY_CAS_I(data.u32, src2_data.u32, src3_data.u32); - data_ready = true; - break; - case B64_TYPE: - case U64_TYPE: - op_result.u64 = MY_CAS_I(data.u64, src2_data.u64, src3_data.u64); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_CAS_I(data.s32, src2_data.s32, src3_data.s32); - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch (%x) with instruction\natom.CAS " - "only accepts b32 and b64\n", - to_type); - assert(0); - break; - } - - break; - } - // EXCH - case ATOMIC_EXCH: { - switch (to_type) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = MY_EXCH(data.u32, src2_data.u32); - data_ready = true; - break; - case B64_TYPE: - case U64_TYPE: - op_result.u64 = MY_EXCH(data.u64, src2_data.u64); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_EXCH(data.s32, src2_data.s32); - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch (%x) with instruction\natom.EXCH " - "only accepts b32\n", - to_type); - assert(0); - break; - } - - break; - } - // ADD - case ATOMIC_ADD: { - switch (to_type) { - case U32_TYPE: - op_result.u32 = data.u32 + src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 + src2_data.s32; - data_ready = true; - break; - case U64_TYPE: - op_result.u64 = data.u64 + src2_data.u64; - data_ready = true; - break; - case F32_TYPE: - op_result.f32 = data.f32 + src2_data.f32; - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch with instruction\natom.ADD only " - "accepts u32, s32, u64, and f32\n"); - assert(0); - break; + break; } + // XOR + case ATOMIC_XOR: + { + + switch ( to_type ) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 ^ src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 ^ src2_data.s32; + data_ready = true; + break; + default: + printf("Execution error: type mismatch (%x) with instruction\natom.XOR only accepts b32\n", to_type); + assert(0); + break; + } - break; - } - // INC - case ATOMIC_INC: { - switch (to_type) { - case U32_TYPE: - op_result.u32 = MY_INC_I(data.u32, src2_data.u32); - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch with instruction\natom.INC only " - "accepts u32 and s32\n"); - assert(0); - break; + break; } + // CAS + case ATOMIC_CAS: + { + + ptx_reg_t src3_data; + const operand_info &src3 = pI->src3(); + src3_data = thread->get_operand_value(src3, dst, to_type, thread, 1); + + switch ( to_type ) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = MY_CAS_I(data.u32, src2_data.u32, src3_data.u32); + data_ready = true; + break; + case B64_TYPE: + case U64_TYPE: + op_result.u64 = MY_CAS_I(data.u64, src2_data.u64, src3_data.u64); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_CAS_I(data.s32, src2_data.s32, src3_data.s32); + data_ready = true; + break; + default: + printf("Execution error: type mismatch (%x) with instruction\natom.CAS only accepts b32 and b64\n", to_type); + assert(0); + break; + } - break; - } - // DEC - case ATOMIC_DEC: { - switch (to_type) { - case U32_TYPE: - op_result.u32 = MY_DEC_I(data.u32, src2_data.u32); - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch with instruction\natom.DEC only " - "accepts u32 and s32\n"); - assert(0); - break; + break; } + // EXCH + case ATOMIC_EXCH: + { + switch ( to_type ) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = MY_EXCH(data.u32, src2_data.u32); + data_ready = true; + break; + case B64_TYPE: + case U64_TYPE: + op_result.u64 = MY_EXCH(data.u64, src2_data.u64); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_EXCH(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf("Execution error: type mismatch (%x) with instruction\natom.EXCH only accepts b32\n", to_type); + assert(0); + break; + } - break; - } - // MIN - case ATOMIC_MIN: { - switch (to_type) { - case U32_TYPE: - op_result.u32 = MY_MIN_I(data.u32, src2_data.u32); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_MIN_I(data.s32, src2_data.s32); - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch with instruction\natom.MIN only " - "accepts u32 and s32\n"); - assert(0); - break; + break; } + // ADD + case ATOMIC_ADD: + { + + switch ( to_type ) { + case U32_TYPE: + op_result.u32 = data.u32 + src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 + src2_data.s32; + data_ready = true; + break; + case U64_TYPE: + op_result.u64 = data.u64 + src2_data.u64; + data_ready = true; + break; + case F32_TYPE: + op_result.f32 = data.f32 + src2_data.f32; + data_ready = true; + break; + default: + printf("Execution error: type mismatch with instruction\natom.ADD only accepts u32, s32, u64, and f32\n"); + assert(0); + break; + } - break; - } - // MAX - case ATOMIC_MAX: { - switch (to_type) { - case U32_TYPE: - op_result.u32 = MY_MAX_I(data.u32, src2_data.u32); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_MAX_I(data.s32, src2_data.s32); - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch with instruction\natom.MAX only " - "accepts u32 and s32\n"); - assert(0); - break; + break; } + // INC + case ATOMIC_INC: + { + switch ( to_type ) { + case U32_TYPE: + op_result.u32 = MY_INC_I(data.u32, src2_data.u32); + data_ready = true; + break; + default: + printf("Execution error: type mismatch with instruction\natom.INC only accepts u32 and s32\n"); + assert(0); + break; + } - break; - } - // DEFAULT - default: { - assert(0); - break; - } - } - - // Write operation result into memory - // (i.e. copy src1_data to dst) - if (data_ready) { - mem->write(effective_address, size / 8, &op_result.s64, thread, pI); - } else { - printf("Execution error: data_ready not set\n"); - assert(0); - } -} - -// atom_impl will now result in a callback being called in mem_ctrl_pop -// (gpu-sim.c) -void atom_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - // SYNTAX - // atom.space.operation.type d, a, b[, c]; (now read in callback) - - // obtain memory space of the operation - memory_space_t space = pI->get_space(); - - // get the memory address - const operand_info &src1 = pI->src1(); - // const operand_info &dst = pI->dst(); // not needed for effective address - // calculation - unsigned i_type = pI->get_type(); - ptx_reg_t src1_data; - src1_data = thread->get_operand_value(src1, src1, i_type, thread, 1); - addr_t effective_address = src1_data.u64; - - addr_t effective_address_final; - - // handle generic memory space by converting it to global - if (space == undefined_space) { - if (whichspace(effective_address) == global_space) { - effective_address_final = generic_to_global(effective_address); - space = global_space; - } else if (whichspace(effective_address) == shared_space) { - unsigned smid = thread->get_hw_sid(); - effective_address_final = generic_to_shared(smid, effective_address); - space = shared_space; - } else { - abort(); - } - } else { - assert(space == global_space || space == shared_space); - effective_address_final = effective_address; - } - - // Check state space - assert(space == global_space || space == shared_space); - - thread->m_last_effective_address = effective_address_final; - thread->m_last_memory_space = space; - thread->m_last_dram_callback.function = atom_callback; - thread->m_last_dram_callback.instruction = pI; -} - -void bar_impl(const ptx_instruction *pIin, ptx_thread_info *thread) { - ptx_instruction *pI = const_cast(pIin); - unsigned bar_op = pI->barrier_op(); - unsigned red_op = pI->get_atomic(); - unsigned ctaid = thread->get_cta_uid(); - - switch (bar_op) { - case SYNC_OPTION: { - if (pI->get_num_operands() > 1) { - const operand_info &op0 = pI->dst(); - const operand_info &op1 = pI->src1(); - ptx_reg_t op0_data; - ptx_reg_t op1_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); - pI->set_bar_count(op1_data.u32); - } else { - const operand_info &op0 = pI->dst(); - ptx_reg_t op0_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); + break; } - break; - } - case ARRIVE_OPTION: { - const operand_info &op0 = pI->dst(); - const operand_info &op1 = pI->src1(); - ptx_reg_t op0_data; - ptx_reg_t op1_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); - pI->set_bar_count(op1_data.u32); - break; - } - case RED_OPTION: { - if (pI->get_num_operands() > 3) { - const operand_info &op1 = pI->src1(); - const operand_info &op2 = pI->src2(); - const operand_info &op3 = pI->src3(); - ptx_reg_t op1_data; - ptx_reg_t op2_data; - ptx_reg_t op3_data; - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - op2_data = thread->get_operand_value(op2, op2, U32_TYPE, thread, 1); - op3_data = thread->get_operand_value(op3, op3, PRED_TYPE, thread, 1); - op3_data.u32 = !(op3_data.pred & 0x0001); - pI->set_bar_id(op1_data.u32); - pI->set_bar_count(op2_data.u32); - switch (red_op) { - case ATOMIC_POPC: - thread->popc_reduction(ctaid, op1_data.u32, op3_data.u32); + // DEC + case ATOMIC_DEC: + { + switch ( to_type ) { + case U32_TYPE: + op_result.u32 = MY_DEC_I(data.u32, src2_data.u32); + data_ready = true; break; - case ATOMIC_AND: - thread->and_reduction(ctaid, op1_data.u32, op3_data.u32); + default: + printf("Execution error: type mismatch with instruction\natom.DEC only accepts u32 and s32\n"); + assert(0); break; - case ATOMIC_OR: - thread->or_reduction(ctaid, op1_data.u32, op3_data.u32); + } + + break; + } + // MIN + case ATOMIC_MIN: + { + switch ( to_type ) { + case U32_TYPE: + op_result.u32 = MY_MIN_I(data.u32, src2_data.u32); + data_ready = true; break; - default: - abort(); + case S32_TYPE: + op_result.s32 = MY_MIN_I(data.s32, src2_data.s32); + data_ready = true; break; - } - } else { - const operand_info &op1 = pI->src1(); - const operand_info &op2 = pI->src2(); - ptx_reg_t op1_data; - ptx_reg_t op2_data; - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - op2_data = thread->get_operand_value(op2, op2, PRED_TYPE, thread, 1); - op2_data.u32 = !(op2_data.pred & 0x0001); - pI->set_bar_id(op1_data.u32); - pI->set_bar_count(thread->get_ntid().x * thread->get_ntid().y * - thread->get_ntid().z); - switch (red_op) { - case ATOMIC_POPC: - thread->popc_reduction(ctaid, op1_data.u32, op2_data.u32); + default: + printf("Execution error: type mismatch with instruction\natom.MIN only accepts u32 and s32\n"); + assert(0); break; - case ATOMIC_AND: - thread->and_reduction(ctaid, op1_data.u32, op2_data.u32); + } + + break; + } + // MAX + case ATOMIC_MAX: + { + switch ( to_type ) { + case U32_TYPE: + op_result.u32 = MY_MAX_I(data.u32, src2_data.u32); + data_ready = true; break; - case ATOMIC_OR: - thread->or_reduction(ctaid, op1_data.u32, op2_data.u32); + case S32_TYPE: + op_result.s32 = MY_MAX_I(data.s32, src2_data.s32); + data_ready = true; break; - default: - abort(); + default: + printf("Execution error: type mismatch with instruction\natom.MAX only accepts u32 and s32\n"); + assert(0); break; - } - } - break; - } - default: - abort(); - break; - } + } - thread->m_last_dram_callback.function = bar_callback; - thread->m_last_dram_callback.instruction = pIin; -} - -void bfe_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - unsigned i_type = pI->get_type(); - unsigned msb = (i_type == U32_TYPE || i_type == S32_TYPE) ? 31 : 63; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t src = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - ptx_reg_t data; - unsigned pos = b.u32 & 0xFF; - unsigned len = c.u32 & 0xFF; - switch (i_type) { - case U32_TYPE: { - unsigned mask; - data.u32 = src.u32 >> pos; - mask = 0xFFFFFFFF >> (32 - len); - data.u32 &= mask; - break; - } - case U64_TYPE: { - unsigned long mask; - data.u64 = src.u64 >> pos; - mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - data.u64 &= mask; - break; - } - case S32_TYPE: { - unsigned mask; - unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (src.s32 >> min) & 0x1; - data.s32 = src.s32 >> pos; - if (sbit > 0) { - mask = 0xFFFFFFFF << len; - data.s32 |= mask; - } else { - mask = 0xFFFFFFFF >> (32 - len); - data.s32 &= mask; + break; } - break; - } - case S64_TYPE: { - unsigned long mask; - unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (src.s64 >> min) & 0x1; - data.s64 = src.s64 >> pos; - if (sbit > 0) { - mask = 0xFFFFFFFFFFFFFFFF << len; - data.s64 |= mask; + // DEFAULT + default: + { + assert(0); + break; + } + } + + // Write operation result into memory + // (i.e. copy src1_data to dst) + if ( data_ready ) { + mem->write(effective_address,size/8,&op_result.s64,thread,pI); + } else { + printf("Execution error: data_ready not set\n"); + assert(0); + } +} + +// atom_impl will now result in a callback being called in mem_ctrl_pop (gpu-sim.c) +void atom_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + // SYNTAX + // atom.space.operation.type d, a, b[, c]; (now read in callback) + + // obtain memory space of the operation + memory_space_t space = pI->get_space(); + + // get the memory address + const operand_info &src1 = pI->src1(); + // const operand_info &dst = pI->dst(); // not needed for effective address calculation + unsigned i_type = pI->get_type(); + ptx_reg_t src1_data; + src1_data = thread->get_operand_value(src1, src1, i_type, thread, 1); + addr_t effective_address = src1_data.u64; + + addr_t effective_address_final; + + // handle generic memory space by converting it to global + if ( space == undefined_space ) { + if( whichspace(effective_address) == global_space ) { + effective_address_final = generic_to_global(effective_address); + space = global_space; + } else if( whichspace(effective_address) == shared_space ) { + unsigned smid = thread->get_hw_sid(); + effective_address_final = generic_to_shared(smid,effective_address); + space = shared_space; } else { - mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - data.s64 &= mask; + abort(); } - break; - } - default: - printf("Operand type not supported for BFE instruction.\n"); - abort(); - return; - } - thread->set_operand_value(dst, data, i_type, thread, pI); + } else { + assert( space == global_space || space == shared_space ); + effective_address_final = effective_address; + } + + // Check state space + assert( space == global_space || space == shared_space ); + + thread->m_last_effective_address = effective_address_final; + thread->m_last_memory_space = space; + thread->m_last_dram_callback.function = atom_callback; + thread->m_last_dram_callback.instruction = pI; +} + +void bar_impl( const ptx_instruction *pIin, ptx_thread_info *thread ) +{ + ptx_instruction * pI = const_cast(pIin); + unsigned bar_op = pI->barrier_op(); + unsigned red_op = pI->get_atomic(); + unsigned ctaid = thread->get_cta_uid(); + + switch(bar_op){ + case SYNC_OPTION: + { + if(pI->get_num_operands()>1){ + const operand_info &op0 = pI->dst(); + const operand_info &op1 = pI->src1(); + ptx_reg_t op0_data; + ptx_reg_t op1_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + pI->set_bar_count(op1_data.u32); + }else{ + const operand_info &op0 = pI->dst(); + ptx_reg_t op0_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + } + break; + } + case ARRIVE_OPTION: + { + const operand_info &op0 = pI->dst(); + const operand_info &op1 = pI->src1(); + ptx_reg_t op0_data; + ptx_reg_t op1_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + pI->set_bar_count(op1_data.u32); + break; + } + case RED_OPTION: + { + if(pI->get_num_operands()>3){ + const operand_info &op1 = pI->src1(); + const operand_info &op2 = pI->src2(); + const operand_info &op3 = pI->src3(); + ptx_reg_t op1_data; + ptx_reg_t op2_data; + ptx_reg_t op3_data; + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + op2_data = thread->get_operand_value(op2, op2, U32_TYPE, thread, 1); + op3_data = thread->get_operand_value(op3, op3, PRED_TYPE, thread, 1); + op3_data.u32=!(op3_data.pred & 0x0001); + pI->set_bar_id(op1_data.u32); + pI->set_bar_count(op2_data.u32); + switch(red_op){ + case ATOMIC_POPC: + thread->popc_reduction(ctaid,op1_data.u32,op3_data.u32); + break; + case ATOMIC_AND: + thread->and_reduction(ctaid,op1_data.u32,op3_data.u32); + break; + case ATOMIC_OR: + thread->or_reduction(ctaid,op1_data.u32,op3_data.u32); + break; + default: + abort(); + break; + } + }else{ + const operand_info &op1 = pI->src1(); + const operand_info &op2 = pI->src2(); + ptx_reg_t op1_data; + ptx_reg_t op2_data; + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + op2_data = thread->get_operand_value(op2, op2, PRED_TYPE, thread, 1); + op2_data.u32=!(op2_data.pred & 0x0001); + pI->set_bar_id(op1_data.u32); + pI->set_bar_count(thread->get_ntid().x * thread->get_ntid().y * thread->get_ntid().z); + switch(red_op){ + case ATOMIC_POPC: + thread->popc_reduction(ctaid,op1_data.u32,op2_data.u32); + break; + case ATOMIC_AND: + thread->and_reduction(ctaid,op1_data.u32,op2_data.u32); + break; + case ATOMIC_OR: + thread->or_reduction(ctaid,op1_data.u32,op2_data.u32); + break; + default: + abort(); + break; + } + } + break; + } + default: + abort(); + break; + } + + thread->m_last_dram_callback.function = bar_callback; + thread->m_last_dram_callback.instruction = pIin; +} + +void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + unsigned i_type = pI->get_type(); + unsigned msb = (i_type == U32_TYPE || i_type == S32_TYPE) ? 31 : 63; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t src = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + ptx_reg_t data; + unsigned pos = b.u32 & 0xFF; + unsigned len = c.u32 & 0xFF; + switch (i_type) + { + case U32_TYPE: + { + unsigned mask; + data.u32 = src.u32 >> pos; + mask = 0xFFFFFFFF >> (32 - len); + data.u32 &= mask; + break; + } + case U64_TYPE: + { + unsigned long mask; + data.u64 = src.u64 >> pos; + mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); + data.u64 &= mask; + break; + } + case S32_TYPE: + { + unsigned mask; + unsigned min = MY_MIN_I(pos + len - 1, msb); + unsigned sbit = len == 0 ? 0 : (src.s32 >> min) & 0x1; + data.s32 = src.s32 >> pos; + if (sbit > 0) + { + mask = 0xFFFFFFFF << len; + data.s32 |= mask; + } + else + { + mask = 0xFFFFFFFF >> (32 - len); + data.s32 &= mask; + } + break; + } + case S64_TYPE: + { + unsigned long mask; + unsigned min = MY_MIN_I(pos + len - 1, msb); + unsigned sbit = len == 0 ? 0 : (src.s64 >> min) & 0x1; + data.s64 = src.s64 >> pos; + if (sbit > 0) + { + mask = 0xFFFFFFFFFFFFFFFF << len; + data.s64 |= mask; + } + else + { + mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); + data.s64 &= mask; + } + break; + } + default: + printf("Operand type not supported for BFE instruction.\n"); + abort(); + return; + } + thread->set_operand_value(dst, data, i_type, thread, pI); } -void bfi_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - int i, max; - ptx_reg_t src1_data, src2_data; - ptx_reg_t src3_data, src4_data, data; - - const operand_info &dst = - pI->dst(); // get operand info of sources and destination - const operand_info &src1 = - pI->src1(); // use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - const operand_info &src4 = pI->src4(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - src4_data = thread->get_operand_value(src4, dst, i_type, thread, 1); - - switch (i_type) { - case B32_TYPE: +void bfi_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { + int i,max; + ptx_reg_t src1_data, src2_data; + ptx_reg_t src3_data, src4_data, data; + + const operand_info &dst = pI->dst(); //get operand info of sources and destination + const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + const operand_info &src4 = pI->src4(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + src4_data = thread->get_operand_value(src4, dst, i_type, thread, 1); + + switch ( i_type ) { + case B32_TYPE: max = 32; break; - case B64_TYPE: + case B64_TYPE: max = 64; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } - data = src2_data; - unsigned pos = src3_data.u32 & 0xFF; - unsigned len = src4_data.u32 & 0xFF; - for (i = 0; i < len && pos + i < max; i++) { - data.u32 = (~((0x00000001) << (pos + i))) & data.u32; - data.u32 = data.u32 | ((src1_data.u32 & ((0x00000001) << (i))) << (pos)); - } - thread->set_operand_value(dst, data, i_type, thread, pI); -} -void bfind_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); + } + data=src2_data; + unsigned pos = src3_data.u32 & 0xFF; + unsigned len = src4_data.u32 & 0xFF; + for(i=0;iset_operand_value(dst, data, i_type, thread, pI); } +void bfind_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void bra_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = - thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void bra_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); - thread->m_branch_taken = true; - thread->set_npc(target_pc); + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void brx_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = - thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void brx_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); - thread->m_branch_taken = true; - thread->set_npc(target_pc); + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void break_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &target = thread->pop_breakaddr(); - ptx_reg_t target_pc = - thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void break_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &target = thread->pop_breakaddr(); + ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); - thread->m_branch_taken = true; - thread->set_npc(target_pc); + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void breakaddr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &target = pI->dst(); - thread->push_breakaddr(target); - assert( - pI->has_pred() == - false); // pdom analysis cannot handle if this instruction is predicated +void breakaddr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &target = pI->dst(); + thread->push_breakaddr(target); + assert(pI->has_pred() == false); // pdom analysis cannot handle if this instruction is predicated } -void brev_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - - unsigned msb; - switch (i_type) { - case B32_TYPE: - msb = 31; - for (unsigned i = 0; i <= msb; i++) { - if ((src1_data.u32 & (1 << i))) data.u32 |= 1 << (msb - i); - } - break; - case B64_TYPE: - msb = 63; - for (unsigned i = 0; i <= msb; i++) { - if ((src1_data.u64 & (1 << i))) data.u64 |= 1 << (msb - i); - } - break; - default: - assert(0); - } - thread->set_operand_value(dst, data, i_type, thread, pI); -} -void brkpt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} - -unsigned trunc(unsigned num, unsigned precision) { - int mask = 1, latest_one = -1; - unsigned data = num; - for (unsigned j = 0; j < sizeof(unsigned) * 8; j++) { - int bit = data & mask; - if (bit == 1) latest_one = j; - data >>= 1; - } - if (latest_one >= precision) { - // round_up is 1 if the most significant truncated digit is a 1, otherwise - // it is 0 - // int round_up = (num & (1 << (latest_one-precision))) >> - // (latest_one-precision); - // unsigned shifted_output = num >> (latest_one-precision+1); - // if shifted_output is a number like 1111, don't round up - // if (shifted_output == (pow(2,precision)-1)) round_up = 0; - // num = shifted_output + round_up; - num >>= (latest_one - precision + 1); - } - return num; -} -void mapping(int thread, int wmma_type, int wmma_layout, int type, int index, - int stride, int &row, int &col, int &assg_offset) { - int offset; - int c_row_offset[] = {0, 8, 0, 8, 4, 12, 4, 12}; - int c_col_offset[] = {0, 0, 8, 8, 0, 0, 8, 8}; - int c_tg_inside_row_offset[] = {0, 1, 0, 1}; - int c_tg_inside_col_offset[] = {0, 0, 2, 2}; - int c_inside_row_offset[] = {0, 0, 2, 2, 0, 0, 2, 2}; - int c_inside_col_offset[] = {0, 1, 0, 1, 4, 5, 4, 5}; - - offset = thread_group_offset(thread, wmma_type, wmma_layout, type, stride); - - if (wmma_type == LOAD_A) { - if (wmma_layout == ROW) { - offset += index + 8 * ((thread % 16) / 8); - } else { - offset += 64 * (index / 4) + index % 4 + 128 * ((thread % 16) / 8); - } - offset = (offset / 16) * stride + offset % 16; - assg_offset = index + 8 * ((thread % 16) / 8); - } else if (wmma_type == LOAD_B) { - if (wmma_layout == ROW) { - offset += 64 * (index / 4) + index % 4 + 128 * ((thread % 16) / 8); - } else { - offset += index + 8 * ((thread % 16) / 8); - } - offset = (offset / 16) * stride + offset % 16; - assg_offset = index + 8 * ((thread % 16) / 8); - } else if (wmma_type == LOAD_C) { - if (type == F16_TYPE) { - row = c_row_offset[thread / 4] + thread % 4; - col = c_col_offset[thread / 4] + index; - } else { - row = c_row_offset[thread / 4] + c_tg_inside_row_offset[thread % 4] + - c_inside_row_offset[index]; - col = c_col_offset[thread / 4] + c_tg_inside_col_offset[thread % 4] + - c_inside_col_offset[index]; - } - assg_offset = index; - } - - if (wmma_type == LOAD_A || wmma_type == LOAD_B) { - if (wmma_layout == ROW) { - row = offset / 16; - col = offset % 16; - } else { - col = offset / 16; - row = offset % 16; - } - } -} +void brev_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); -void mma_impl(const ptx_instruction *pI, core_t *core, warp_inst_t inst) { - int i, j, k, thrd; - int row, col, offset; - ptx_reg_t matrix_a[16][16]; - ptx_reg_t matrix_b[16][16]; - ptx_reg_t matrix_c[16][16]; - ptx_reg_t matrix_d[16][16]; - ptx_reg_t src_data; - ptx_thread_info *thread; - - unsigned a_layout = pI->get_wmma_layout(0); - unsigned b_layout = pI->get_wmma_layout(1); - unsigned type = pI->get_type(); - unsigned type2 = pI->get_type2(); - int tid; - const operand_info &dst = pI->operand_lookup(0); - - if (core->get_gpu()->is_functional_sim()) - tid = inst.warp_id_func() * core->get_warp_size(); - else - tid = inst.warp_id() * core->get_warp_size(); - float temp; - half temp2; - - for (thrd = 0; thrd < core->get_warp_size(); thrd++) { - thread = core->get_thread_info()[tid + thrd]; - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("THREAD=%d\n:", thrd); - for (int operand_num = 1; operand_num <= 3; operand_num++) { - const operand_info &src_a = pI->operand_lookup(operand_num); - unsigned nelem = src_a.get_vect_nelem(); - ptx_reg_t v[8]; - thread->get_vector_operand_values(src_a, v, nelem); - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - printf("Thread%d_Iteration=%d\n:", thrd, operand_num); - for (k = 0; k < nelem; k++) { - printf("%llx ", v[k].u64); - } - printf("\n"); - } - ptx_reg_t nw_v[16]; - int hex_val; - - if (!((operand_num == 3) && (type2 == F32_TYPE))) { - for (k = 0; k < 2 * nelem; k++) { - if (k % 2 == 1) - hex_val = (v[k / 2].s64 & 0xffff); - else - hex_val = ((v[k / 2].s64 & 0xffff0000) >> 16); - nw_v[k].f16 = *((half *)&hex_val); - } - } - if (!((operand_num == 3) && (type2 == F32_TYPE))) { - for (k = 0; k < 2 * nelem; k++) { - temp = nw_v[k].f16; - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("%.2f ", temp); - } - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) printf("\n"); - } else { - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - for (k = 0; k < 8; k++) { - printf("%.2f ", v[k].f32); - } - printf("\n"); - } - } - switch (operand_num) { - case 1: // operand 1 - for (k = 0; k < 8; k++) { - mapping(thrd, LOAD_A, a_layout, F16_TYPE, k, 16, row, col, offset); - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("A:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, - offset); - matrix_a[row][col] = nw_v[offset]; - } - break; - case 2: // operand 2 - for (k = 0; k < 8; k++) { - mapping(thrd, LOAD_B, b_layout, F16_TYPE, k, 16, row, col, offset); - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("B:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, - offset); - matrix_b[row][col] = nw_v[offset]; - } - break; - case 3: // operand 3 - for (k = 0; k < 8; k++) { - mapping(thrd, LOAD_C, ROW, type2, k, 16, row, col, offset); - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("C:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, - offset); - if (type2 != F16_TYPE) { - matrix_c[row][col] = v[offset]; - } else { - matrix_c[row][col] = nw_v[offset]; + unsigned msb; + switch(i_type){ + case B32_TYPE: + msb = 31; + for (unsigned i=0; i<=msb; i++) { + if((src1_data.u32 & (1 << i))) + data.u32 |= 1 << (msb - i); } - } - break; - default: - printf("Invalid Operand Index\n"); - } - } - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) printf("\n"); - } - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - printf("MATRIX_A\n"); - for (i = 0; i < 16; i++) { - for (j = 0; j < 16; j++) { - temp = matrix_a[i][j].f16; - printf("%.2f ", temp); - } - printf("\n"); - } - printf("MATRIX_B\n"); - for (i = 0; i < 16; i++) { - for (j = 0; j < 16; j++) { - temp = matrix_b[i][j].f16; - printf("%.2f ", temp); - } - printf("\n"); - } - printf("MATRIX_C\n"); - for (i = 0; i < 16; i++) { - for (j = 0; j < 16; j++) { - if (type2 == F16_TYPE) { - temp = matrix_c[i][j].f16; - printf("%.2f ", temp); - } else - printf("%.2f ", matrix_c[i][j].f32); - } - printf("\n"); - } - } - for (i = 0; i < 16; i++) { - for (j = 0; j < 16; j++) { - matrix_d[i][j].f16 = 0; - } - } - - for (i = 0; i < 16; i++) { - for (j = 0; j < 16; j++) { - for (k = 0; k < 16; k++) { - matrix_d[i][j].f16 = - matrix_d[i][j].f16 + matrix_a[i][k].f16 * matrix_b[k][j].f16; - } - if ((type == F16_TYPE) && (type2 == F16_TYPE)) - matrix_d[i][j].f16 += matrix_c[i][j].f16; - else if ((type == F32_TYPE) && (type2 == F16_TYPE)) { - temp2 = matrix_d[i][j].f16 + matrix_c[i][j].f16; - temp = temp2; - matrix_d[i][j].f32 = temp; - } else if ((type == F16_TYPE) && (type2 == F32_TYPE)) { - temp = matrix_d[i][j].f16; - temp += matrix_c[i][j].f32; - matrix_d[i][j].f16 = half(temp); - } else { - temp = matrix_d[i][j].f16; - temp += matrix_c[i][j].f32; - matrix_d[i][j].f32 = temp; - } - } - } - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - printf("MATRIX_D\n"); - for (i = 0; i < 16; i++) { - for (j = 0; j < 16; j++) { - if (type == F16_TYPE) { - temp = matrix_d[i][j].f16; - printf("%.2f ", temp); - } else - printf("%.2f ", matrix_d[i][j].f32); - } - printf("\n"); - } - } - for (thrd = 0; thrd < core->get_warp_size(); thrd++) { - int row_t[8]; - int col_t[8]; - for (k = 0; k < 8; k++) { - mapping(thrd, LOAD_C, ROW, type, k, 16, row_t[k], col_t[k], offset); - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma:store:row:%d,col%d\n", row_t[k], col_t[k]); - } - thread = core->get_thread_info()[tid + thrd]; - - if (type == F32_TYPE) { - thread->set_wmma_vector_operand_values( - dst, matrix_d[row_t[0]][col_t[0]], matrix_d[row_t[1]][col_t[1]], - matrix_d[row_t[2]][col_t[2]], matrix_d[row_t[3]][col_t[3]], - matrix_d[row_t[4]][col_t[4]], matrix_d[row_t[5]][col_t[5]], - matrix_d[row_t[6]][col_t[6]], matrix_d[row_t[7]][col_t[7]]); - - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - printf("thread%d:", thrd); - for (k = 0; k < 8; k++) { - printf("%.2f ", matrix_d[row_t[k]][col_t[k]].f32); - } - printf("\n"); - } - } else if (type == F16_TYPE) { - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - printf("thread%d:", thrd); - for (k = 0; k < 8; k++) { - temp = matrix_d[row_t[k]][col_t[k]].f16; - printf("%.2f ", temp); - } - printf("\n"); - - printf("thread%d:", thrd); - for (k = 0; k < 8; k++) { - printf("%x ", (unsigned int)matrix_d[row_t[k]][col_t[k]].f16); - } - printf("\n"); - } - ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4; - nw_data1.s64 = ((matrix_d[row_t[0]][col_t[0]].s64 & 0xffff)) | - ((matrix_d[row_t[1]][col_t[1]].s64 & 0xffff) << 16); - nw_data2.s64 = ((matrix_d[row_t[2]][col_t[2]].s64 & 0xffff)) | - ((matrix_d[row_t[3]][col_t[3]].s64 & 0xffff) << 16); - nw_data3.s64 = ((matrix_d[row_t[4]][col_t[4]].s64 & 0xffff)) | - ((matrix_d[row_t[5]][col_t[5]].s64 & 0xffff) << 16); - nw_data4.s64 = ((matrix_d[row_t[6]][col_t[6]].s64 & 0xffff)) | - ((matrix_d[row_t[7]][col_t[7]].s64 & 0xffff) << 16); - thread->set_vector_operand_values(dst, nw_data1, nw_data2, nw_data3, - nw_data4); - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("thread%d=%llx,%llx,%llx,%llx", thrd, nw_data1.s64, nw_data2.s64, - nw_data3.s64, nw_data4.s64); - - } else { - printf("wmma:mma:wrong type\n"); - abort(); + break; + case B64_TYPE: + msb = 63; + for (unsigned i=0; i<=msb; i++) { + if((src1_data.u64 & (1 << i))) + data.u64 |= 1 << (msb - i); + } + break; + default: assert(0); } - } + thread->set_operand_value(dst,data, i_type, thread, pI); } +void brkpt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void call_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - static unsigned call_uid_next = 1; - - const operand_info &target = pI->func_addr(); - assert(target.is_function_address()); - const symbol *func_addr = target.get_symbol(); - function_info *target_func = func_addr->get_pc(); - if (target_func->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", - target_func->get_name().c_str()); - } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", - target_func->get_name().c_str()); - /* - * Some of the instructions like printf() gives the gpgpusim the wrong - * impression that it is a function call. - * As printf() doesnt have a body like functions do, doing pdom analysis for - * printf() causes a crash. - */ - if (target_func->get_function_size() > 0) target_func->do_pdom(); - target_func->set_pdom(); - } - - // check that number of args and return match function requirements - if (pI->has_return() ^ target_func->has_return()) { - printf( - "GPGPU-Sim PTX: Execution error - mismatch in number of return values " - "between\n" - " call instruction and function declaration\n"); - abort(); - } - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - unsigned n_operands = pI->get_num_operands(); - - if (n_operands != (n_return + 1 + n_args)) { - printf( - "GPGPU-Sim PTX: Execution error - mismatch in number of arguements " - "between\n" - " call instruction and function declaration\n"); - abort(); - } +unsigned trunc(unsigned num, unsigned precision) { + int mask = 1, latest_one = -1; + unsigned data = num; + for (unsigned j = 0; j < sizeof(unsigned)*8; j++) { + int bit = data & mask; + if (bit == 1) latest_one = j; + data >>= 1; + } + if (latest_one >= precision) { + // round_up is 1 if the most significant truncated digit is a 1, otherwise it is 0 + //int round_up = (num & (1 << (latest_one-precision))) >> (latest_one-precision); + //unsigned shifted_output = num >> (latest_one-precision+1); + // if shifted_output is a number like 1111, don't round up + //if (shifted_output == (pow(2,precision)-1)) round_up = 0; + //num = shifted_output + round_up; + num >>= (latest_one-precision+1); + } + return num; +} +void mapping(int thread,int wmma_type,int wmma_layout,int type,int index,int stride,int &row,int &col,int &assg_offset){ + int offset; + int c_row_offset[]={0,8,0,8,4,12,4,12}; + int c_col_offset[]={0,0,8,8,0,0,8,8}; + int c_tg_inside_row_offset[]={0,1,0,1}; + int c_tg_inside_col_offset[]={0,0,2,2}; + int c_inside_row_offset[]={0,0,2,2,0,0,2,2}; + int c_inside_col_offset[]={0,1,0,1,4,5,4,5}; + + offset=thread_group_offset(thread,wmma_type,wmma_layout,type,stride); + + if(wmma_type==LOAD_A){ + if(wmma_layout==ROW){ + offset+=index+8*((thread%16)/8); + } + else{ + offset+=64*(index/4)+index%4+128*((thread%16)/8); + } + offset=(offset/16)*stride+offset%16; + assg_offset=index+8*((thread%16)/8); + } + else if(wmma_type==LOAD_B){ + if(wmma_layout==ROW){ + offset+=64*(index/4)+index%4+128*((thread%16)/8); + } + else{ + offset+=index+8*((thread%16)/8); + } + offset=(offset/16)*stride+offset%16; + assg_offset=index+8*((thread%16)/8); + } + else if( wmma_type==LOAD_C){ + if(type==F16_TYPE){ + row=c_row_offset[thread/4]+thread%4; + col=c_col_offset[thread/4]+index; + } + else{ + row=c_row_offset[thread/4]+c_tg_inside_row_offset[thread%4]+c_inside_row_offset[index]; + col=c_col_offset[thread/4]+c_tg_inside_col_offset[thread%4]+c_inside_col_offset[index]; + } + assg_offset=index; + } + + if(wmma_type==LOAD_A||wmma_type==LOAD_B){ + if(wmma_layout==ROW){ + row=offset/16; + col=offset%16; + } + else{ + col=offset/16; + row=offset%16; + } + } +} + +void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) +{ + int i,j,k,thrd; + int row,col,offset; + ptx_reg_t matrix_a[16][16]; + ptx_reg_t matrix_b[16][16]; + ptx_reg_t matrix_c[16][16]; + ptx_reg_t matrix_d[16][16]; + ptx_reg_t src_data; + ptx_thread_info *thread; + + unsigned a_layout = pI->get_wmma_layout(0); + unsigned b_layout = pI->get_wmma_layout(1); + unsigned type = pI->get_type(); + unsigned type2 = pI->get_type2(); + int tid ; + const operand_info &dst = pI->operand_lookup(0); + + if(core->get_gpu()->is_functional_sim()) + tid= inst.warp_id_func()*core->get_warp_size(); + else + tid= inst.warp_id()*core->get_warp_size(); + float temp; + half temp2; + + for (thrd=0; thrd < core->get_warp_size(); thrd++){ + thread = core->get_thread_info()[tid+thrd]; + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("THREAD=%d\n:",thrd); + for(int operand_num=1;operand_num<=3;operand_num++){ + const operand_info &src_a= pI->operand_lookup(operand_num); + unsigned nelem = src_a.get_vect_nelem(); + ptx_reg_t v[8]; + thread->get_vector_operand_values( src_a, v, nelem ); + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ + printf("Thread%d_Iteration=%d\n:", thrd, operand_num); + for(k = 0; k < nelem; k++){ + printf("%llx ",v[k].u64); + } + printf("\n"); + } + ptx_reg_t nw_v[16]; + int hex_val; + + if(!((operand_num==3)&&(type2==F32_TYPE))){ + for(k=0;k<2*nelem;k++){ + if(k%2==1) + hex_val=(v[k/2].s64&0xffff); + else + hex_val=((v[k/2].s64&0xffff0000)>>16); + nw_v[k].f16 =*((half *)&hex_val); + } + } + if(!((operand_num==3)&&(type2==F32_TYPE))){ + for(k=0;k<2*nelem;k++){ + temp=nw_v[k].f16; + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("%.2f ",temp); + } + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("\n"); + } + else{ + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ + for(k=0;k<8;k++){ + printf("%.2f ",v[k].f32); + } + printf("\n"); + } + } + switch(operand_num) { + case 1 ://operand 1 + for(k=0;k<8;k++){ + mapping(thrd,LOAD_A,a_layout,F16_TYPE,k,16,row,col,offset); + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("A:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); + matrix_a[row][col]=nw_v[offset]; + } + break; + case 2 ://operand 2 + for(k=0;k<8;k++){ + mapping(thrd,LOAD_B,b_layout,F16_TYPE,k,16,row,col,offset); + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("B:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); + matrix_b[row][col]=nw_v[offset]; + } + break; + case 3 ://operand 3 + for(k=0;k<8;k++){ + mapping(thrd,LOAD_C,ROW,type2,k,16,row,col,offset); + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("C:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); + if(type2!=F16_TYPE){ + matrix_c[row][col]=v[offset]; + } + else { + matrix_c[row][col]=nw_v[offset]; + } + } + break; + default : + printf("Invalid Operand Index\n" ); + } + } + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("\n"); + } + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ + printf("MATRIX_A\n"); + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + temp=matrix_a[i][j].f16; + printf("%.2f ",temp); + } + printf("\n"); + } + printf("MATRIX_B\n"); + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + temp=matrix_b[i][j].f16; + printf("%.2f ",temp); + } + printf("\n"); + } + printf("MATRIX_C\n"); + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + if(type2==F16_TYPE){ + temp=matrix_c[i][j].f16; + printf("%.2f ",temp); + } + else + printf("%.2f ",matrix_c[i][j].f32); + } + printf("\n"); + } + } + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + matrix_d[i][j].f16=0; + } + } + + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + for(k=0;k<16;k++){ + matrix_d[i][j].f16=matrix_d[i][j].f16+matrix_a[i][k].f16*matrix_b[k][j].f16; + } + if((type==F16_TYPE)&&(type2==F16_TYPE)) + matrix_d[i][j].f16+=matrix_c[i][j].f16; + else if((type==F32_TYPE)&&(type2==F16_TYPE)){ + temp2=matrix_d[i][j].f16+matrix_c[i][j].f16; + temp=temp2; + matrix_d[i][j].f32=temp; + } + else if((type==F16_TYPE)&&(type2==F32_TYPE)){ + temp=matrix_d[i][j].f16; + temp+=matrix_c[i][j].f32; + matrix_d[i][j].f16=half(temp); + } + else{ + temp=matrix_d[i][j].f16; + temp+=matrix_c[i][j].f32; + matrix_d[i][j].f32=temp; + } + } + } + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ + printf("MATRIX_D\n"); + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + if(type==F16_TYPE){ + temp=matrix_d[i][j].f16; + printf("%.2f ",temp); + } + else + printf("%.2f ",matrix_d[i][j].f32); + } + printf("\n"); + } + } + for (thrd=0; thrd < core->get_warp_size(); thrd++){ + int row_t[8]; + int col_t[8]; + for(k=0;k<8;k++){ + mapping(thrd,LOAD_C,ROW,type,k,16,row_t[k],col_t[k],offset); + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma:store:row:%d,col%d\n",row_t[k],col_t[k]); + } + thread = core->get_thread_info()[tid+thrd]; + + + if(type==F32_TYPE){ + thread->set_wmma_vector_operand_values(dst,matrix_d[row_t[0]][col_t[0]],matrix_d[row_t[1]][col_t[1]],matrix_d[row_t[2]][col_t[2]],matrix_d[row_t[3]][col_t[3]],matrix_d[row_t[4]][col_t[4]],matrix_d[row_t[5]][col_t[5]],matrix_d[row_t[6]][col_t[6]],matrix_d[row_t[7]][col_t[7]]); + + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + { + printf("thread%d:",thrd); + for(k=0;k<8;k++){ + printf("%.2f ",matrix_d[row_t[k]][col_t[k]].f32); + } + printf("\n"); + } + } + else if(type==F16_TYPE){ + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ + printf("thread%d:",thrd); + for(k=0;k<8;k++){ + temp=matrix_d[row_t[k]][col_t[k]].f16; + printf("%.2f ",temp); + } + printf("\n"); + + printf("thread%d:",thrd); + for(k=0;k<8;k++){ + printf("%x ", (unsigned int)matrix_d[row_t[k]][col_t[k]].f16); + } + printf("\n"); + } + ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4; + nw_data1.s64=((matrix_d[row_t[0]][col_t[0]].s64 & 0xffff))|((matrix_d[row_t[1]][col_t[1]].s64&0xffff)<<16); + nw_data2.s64=((matrix_d[row_t[2]][col_t[2]].s64 & 0xffff))|((matrix_d[row_t[3]][col_t[3]].s64&0xffff)<<16); + nw_data3.s64=((matrix_d[row_t[4]][col_t[4]].s64 & 0xffff))|((matrix_d[row_t[5]][col_t[5]].s64&0xffff)<<16); + nw_data4.s64=((matrix_d[row_t[6]][col_t[6]].s64 & 0xffff))|((matrix_d[row_t[7]][col_t[7]].s64&0xffff)<<16); + thread->set_vector_operand_values(dst,nw_data1,nw_data2,nw_data3,nw_data4); + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("thread%d=%llx,%llx,%llx,%llx", thrd, nw_data1.s64, nw_data2.s64, nw_data3.s64, nw_data4.s64); + + } + else{ + printf("wmma:mma:wrong type\n"); + abort(); + } + } +} + +void call_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + static unsigned call_uid_next = 1; + + const operand_info &target = pI->func_addr(); + assert( target.is_function_address() ); + const symbol *func_addr = target.get_symbol(); + function_info *target_func = func_addr->get_pc(); + if (target_func->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", target_func->get_name().c_str() ); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", target_func->get_name().c_str() ); + /* + * Some of the instructions like printf() gives the gpgpusim the wrong impression that it is a function call. + * As printf() doesnt have a body like functions do, doing pdom analysis for printf() causes a crash. + */ + if (target_func->get_function_size() >0) + target_func->do_pdom(); + target_func->set_pdom(); + } + + // check that number of args and return match function requirements + if( pI->has_return() ^ target_func->has_return() ) { + printf("GPGPU-Sim PTX: Execution error - mismatch in number of return values between\n" + " call instruction and function declaration\n"); + abort(); + } + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + unsigned n_operands = pI->get_num_operands(); + + if( n_operands != (n_return+1+n_args) ) { + printf("GPGPU-Sim PTX: Execution error - mismatch in number of arguements between\n" + " call instruction and function declaration\n"); + abort(); + } + + // handle intrinsic functions + std::string fname = target_func->get_name(); + if( fname == "vprintf" ) { + gpgpusim_cuda_vprintf(pI, thread, target_func); + return; + } - // handle intrinsic functions - std::string fname = target_func->get_name(); - if (fname == "vprintf") { - gpgpusim_cuda_vprintf(pI, thread, target_func); - return; - } #if (CUDART_VERSION >= 5000) - // Jin: handle device runtime apis for CDP - else if (fname == "cudaGetParameterBufferV2") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_getParameterBufferV2( - pI, thread, target_func); - return; - } else if (fname == "cudaLaunchDeviceV2") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_launchDeviceV2( - pI, thread, target_func); - return; - } else if (fname == "cudaStreamCreateWithFlags") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_streamCreateWithFlags( - pI, thread, target_func); - return; - } + //Jin: handle device runtime apis for CDP + else if(fname == "cudaGetParameterBufferV2") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_getParameterBufferV2(pI, thread, target_func); + return; + } + else if(fname == "cudaLaunchDeviceV2") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_launchDeviceV2(pI, thread, target_func); + return; + } + else if(fname == "cudaStreamCreateWithFlags") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_streamCreateWithFlags(pI, thread, target_func); + return; + } #endif - // read source arguements into register specified in declaration of function - arg_buffer_list_t arg_values; - copy_args_into_buffer_list(pI, thread, target_func, arg_values); + // read source arguements into register specified in declaration of function + arg_buffer_list_t arg_values; + copy_args_into_buffer_list(pI, thread, target_func, arg_values); - // record local for return value (we only support a single return value) - const symbol *return_var_src = NULL; - const symbol *return_var_dst = NULL; - if (target_func->has_return()) { - return_var_dst = pI->dst().get_symbol(); - return_var_src = target_func->get_return_var(); - } + // record local for return value (we only support a single return value) + const symbol *return_var_src = NULL; + const symbol *return_var_dst = NULL; + if( target_func->has_return() ) { + return_var_dst = pI->dst().get_symbol(); + return_var_src = target_func->get_return_var(); + } - gpgpu_sim *gpu = thread->get_gpu(); - unsigned callee_pc = 0, callee_rpc = 0; - if (gpu->simd_model() == POST_DOMINATOR) { - thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(), - &callee_pc, &callee_rpc); - assert(callee_pc == thread->get_pc()); - } + gpgpu_sim *gpu = thread->get_gpu(); + unsigned callee_pc=0, callee_rpc=0; + if( gpu->simd_model() == POST_DOMINATOR ) { + thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(),&callee_pc,&callee_rpc); + assert( callee_pc == thread->get_pc() ); + } - thread->callstack_push(callee_pc + pI->inst_size(), callee_rpc, - return_var_src, return_var_dst, call_uid_next++); + thread->callstack_push(callee_pc + pI->inst_size(), callee_rpc, return_var_src, return_var_dst, call_uid_next++); - copy_buffer_list_into_frame(thread, arg_values); + copy_buffer_list_into_frame(thread, arg_values); - thread->set_npc(target_func); + thread->set_npc(target_func); } -// Ptxplus version of call instruction. Jumps to a label not a different Kernel. -void callp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - static unsigned call_uid_next = 1; +//Ptxplus version of call instruction. Jumps to a label not a different Kernel. +void callp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + + static unsigned call_uid_next = 1; - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = - thread->get_operand_value(target, target, U32_TYPE, thread, 1); + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); - const symbol *return_var_src = NULL; - const symbol *return_var_dst = NULL; + const symbol *return_var_src = NULL; + const symbol *return_var_dst = NULL; - gpgpu_sim *gpu = thread->get_gpu(); - unsigned callee_pc = 0, callee_rpc = 0; - if (gpu->simd_model() == POST_DOMINATOR) { - thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(), - &callee_pc, &callee_rpc); - assert(callee_pc == thread->get_pc()); - } + gpgpu_sim *gpu = thread->get_gpu(); + unsigned callee_pc=0, callee_rpc=0; + if( gpu->simd_model() == POST_DOMINATOR ) { + thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(),&callee_pc,&callee_rpc); + assert( callee_pc == thread->get_pc() ); + } - thread->callstack_push_plus(callee_pc + pI->inst_size(), callee_rpc, - return_var_src, return_var_dst, call_uid_next++); - thread->set_npc(target_pc); + thread->callstack_push_plus(callee_pc + pI->inst_size(), callee_rpc, return_var_src, return_var_dst, call_uid_next++); + thread->set_npc(target_pc); } -void clz_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void clz_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - int max; - unsigned long long mask; - d.u64 = 0; + int max; + unsigned long long mask; + d.u64 = 0; - switch (i_type) { - case B32_TYPE: + switch ( i_type ) { + case B32_TYPE: max = 32; mask = 0x80000000; break; - case B64_TYPE: + case B64_TYPE: max = 64; mask = 0x8000000000000000; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - while ((d.u32 < max) && ((a.u64 & mask) == 0)) { - d.u32++; - a.u64 = a.u64 << 1; - } + while ((d.u32 < max) && ((a.u64&mask) == 0) ) { + d.u32++; + a.u64 = a.u64 << 1; + } - thread->set_operand_value(dst, d, B32_TYPE, thread, pI); + thread->set_operand_value(dst,d, B32_TYPE, thread, pI); } -void cnot_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void cnot_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch (i_type) { - case PRED_TYPE: - d.pred = ((a.pred & 0x0001) == 0) ? 1 : 0; - break; - case B16_TYPE: - d.u16 = (a.u16 == 0) ? 1 : 0; - break; - case B32_TYPE: - d.u32 = (a.u32 == 0) ? 1 : 0; - break; - case B64_TYPE: - d.u64 = (a.u64 == 0) ? 1 : 0; - break; - default: + switch ( i_type ) { + case PRED_TYPE: d.pred = ((a.pred & 0x0001) == 0)?1:0; break; + case B16_TYPE: d.u16 = (a.u16 == 0)?1:0; break; + case B32_TYPE: d.u32 = (a.u32 == 0)?1:0; break; + case B64_TYPE: d.u64 = (a.u64 == 0)?1:0; break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -void cos_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void cos_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch (i_type) { - case F32_TYPE: + + switch ( i_type ) { + case F32_TYPE: d.f32 = cos(a.f32); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - - thread->set_operand_value(dst, d, i_type, thread, pI); -} + assert(0); + break; + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +ptx_reg_t chop( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + switch ( to_width ) { + case 8: x.mask_and(0,0xFF); break; + case 16: x.mask_and(0,0xFFFF); break; + case 32: x.mask_and(0,0xFFFFFFFF); break; + case 64: break; + default: assert(0); + } + return x; +} + +ptx_reg_t sext( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + x=chop(x,0,from_width,0,rounding_mode,saturation_mode); + switch ( from_width ) { + case 8: if ( x.get_bit(7) ) x.mask_or(0xFFFFFFFF,0xFFFFFF00);break; + case 16:if ( x.get_bit(15) ) x.mask_or(0xFFFFFFFF,0xFFFF0000);break; + case 32: if ( x.get_bit(31) ) x.mask_or(0xFFFFFFFF,0x00000000);break; + case 64: break; + default: assert(0); + } + return x; +} + +// sign extend depending on the destination register size - hack to get SobelFilter working in CUDA 4.2 +ptx_reg_t sexd( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + x=chop(x,0,from_width,0,rounding_mode,saturation_mode); + switch ( to_width ) { + case 8: if ( x.get_bit(7) ) x.mask_or(0xFFFFFFFF,0xFFFFFF00);break; + case 16:if ( x.get_bit(15) ) x.mask_or(0xFFFFFFFF,0xFFFF0000);break; + case 32: if ( x.get_bit(31) ) x.mask_or(0xFFFFFFFF,0x00000000);break; + case 64: break; + default: assert(0); + } + return x; +} + +ptx_reg_t zext( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + return chop(x,0,from_width,0,rounding_mode,saturation_mode); +} + +int saturatei(int a, int max, int min) +{ + if (a > max) a = max; + else if (a < min) a = min; + return a; +} + +unsigned int saturatei(unsigned int a, unsigned int max) +{ + if (a > max) a = max; + return a; +} + +ptx_reg_t f2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + half mytemp; + half_float::half tmp_h; + //assert( from_width == 32); + + enum cudaRoundMode mode = cudaRoundZero; + switch (rounding_mode) { + case RZI_OPTION: mode = cudaRoundZero; break; + case RNI_OPTION: mode = cudaRoundNearest; break; + case RMI_OPTION: mode = cudaRoundMinInf; break; + case RPI_OPTION: mode = cudaRoundPosInf; break; + default: break; + } + + ptx_reg_t y; + if ( to_sign == 1 ) { // convert to 64-bit number first? + int tmp = cuda_math::float2int(x.f32, mode); + if ((x.u32 & 0x7f800000) == 0) + tmp = 0; // round denorm. FP to 0 + if (saturation_mode && to_width < 32) { + tmp = saturatei(tmp, (1<::round_style>(x.f32);//mytemp; + break; + case 32: + y.f32=float(x.f16); + break; // handled by f2f + case 64: + y.f64 = x.f32; + break; + default: assert(0); break; + } + } + return y; +} + +double saturated2i (double a, double max, double min) { + if (a > max) a = max; + else if (a < min) a = min; + return a; +} + +ptx_reg_t d2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + assert( from_width == 64); + + double tmp; + switch (rounding_mode) { + case RZI_OPTION: tmp = trunc(x.f64); break; + case RNI_OPTION: tmp = nearbyint(x.f64); break; + case RMI_OPTION: tmp = floor(x.f64); break; + case RPI_OPTION: tmp = ceil(x.f64); break; + default: tmp = x.f64; break; + } + + ptx_reg_t y; + if ( to_sign == 1 ) { + tmp = saturated2i(tmp, ((1<<(to_width - 1)) - 1), (1<<(to_width - 1)) ); + switch ( to_width ) { + case 8: y.s8 = (char)tmp; break; + case 16: y.s16 = (short)tmp; break; + case 32: y.s32 = (int)tmp; break; + case 64: y.s64 = (long long)tmp; break; + default: assert(0); break; + } + } else if ( to_sign == 0 ) { + tmp = saturated2i(tmp, ((1<<(to_width - 1)) - 1), 0); + switch ( to_width ) { + case 8: y.u8 = (unsigned char)tmp; break; + case 16: y.u16 = (unsigned short)tmp; break; + case 32: y.u32 = (unsigned int)tmp; break; + case 64: y.u64 = (unsigned long long)tmp; break; + default: assert(0); break; + } + } else { + switch ( to_width ) { + case 16: assert(0); break; + case 32: + y.f32 = x.f64; + break; + case 64: + y.f64 = x.f64; // should be handled by d2d + break; + default: assert(0); break; + } + } + return y; +} + +ptx_reg_t s2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + ptx_reg_t y; + + if (from_width < 64) { // 32-bit conversion + y = sext(x,from_width,32,0,rounding_mode,saturation_mode); + + switch ( to_width ) { + case 16: assert(0); break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: y.f32 = cuda_math::__int2float_rz(y.s32); break; + case RN_OPTION: y.f32 = cuda_math::__int2float_rn(y.s32); break; + case RM_OPTION: y.f32 = cuda_math::__int2float_rd(y.s32); break; + case RP_OPTION: y.f32 = cuda_math::__int2float_ru(y.s32); break; + default: break; + } + break; + case 64: y.f64 = y.s32; break; // no rounding needed + default: assert(0); break; + } + } else { + switch ( to_width ) { + case 16: assert(0); break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: y.f32 = cuda_math::__ll2float_rz(y.s64); break; + case RN_OPTION: y.f32 = cuda_math::__ll2float_rn(y.s64); break; + case RM_OPTION: y.f32 = cuda_math::__ll2float_rd(y.s64); break; + case RP_OPTION: y.f32 = cuda_math::__ll2float_ru(y.s64); break; + default: break; + } + break; + case 64: y.f64 = y.s64; break; // no internal implementation found + default: assert(0); break; + } + } + + // saturating an integer to 1 or 0? + return y; +} + +ptx_reg_t u2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + ptx_reg_t y; + + if (from_width < 64) { // 32-bit conversion + y = zext(x,from_width,32,0,rounding_mode,saturation_mode); + + switch ( to_width ) { + case 16: assert(0); break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: y.f32 = cuda_math::__uint2float_rz(y.u32); break; + case RN_OPTION: y.f32 = cuda_math::__uint2float_rn(y.u32); break; + case RM_OPTION: y.f32 = cuda_math::__uint2float_rd(y.u32); break; + case RP_OPTION: y.f32 = cuda_math::__uint2float_ru(y.u32); break; + default: break; + } + break; + case 64: y.f64 = y.u32; break; // no rounding needed + default: assert(0); break; + } + } else { + switch ( to_width ) { + case 16: assert(0); break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; + case RN_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; + case RM_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; + case RP_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; + default: break; + } + break; + case 64: y.f64 = y.u64; break; // no internal implementation found + default: assert(0); break; + } + } + + // saturating an integer to 1 or 0? + return y; +} + +ptx_reg_t f2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + ptx_reg_t y; + if (from_width == 16){ + half_float::detail::uint16 val = x.u16; + y.f32 = half_float::detail::half2float(val); + }else{ + switch ( rounding_mode ) { + case RZI_OPTION: + y.f32 = truncf(x.f32); + break; + case RNI_OPTION: + #if CUDART_VERSION >= 3000 + y.f32 = nearbyintf(x.f32); + #else + y.f32 = cuda_math::__internal_nearbyintf(x.f32); + #endif + break; + case RMI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = floorf(x.f32); + } + break; + case RPI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = ceilf(x.f32); + } + break; + default: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = x.f32; + } + break; + } + #if CUDART_VERSION >= 3000 + if (isnanf(y.f32)) + #else + if (cuda_math::__cuda___isnanf(y.f32)) + #endif + { + y.u32 = 0x7fffffff; + } else if (saturation_mode) { + y.f32 = cuda_math::__saturatef(y.f32); + } + } + + return y; +} + +ptx_reg_t d2d( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + ptx_reg_t y; + switch ( rounding_mode ) { + case RZI_OPTION: + y.f64 = trunc(x.f64); + break; + case RNI_OPTION: +#if CUDART_VERSION >= 3000 + y.f64 = nearbyint(x.f64); +#else + y.f64 = cuda_math::__internal_nearbyintf(x.f64); +#endif + break; + case RMI_OPTION: + y.f64 = floor(x.f64); + break; + case RPI_OPTION: + y.f64 = ceil(x.f64); + break; + default: + y.f64 = x.f64; + break; + } + if (std::isnan(y.f64)) { + y.u64 = 0xfff8000000000000ull; + } else if (saturation_mode) { + y.f64 = cuda_math::__saturatef(y.f64); + } + return y; +} + +ptx_reg_t (*g_cvt_fn[11][11])( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode ) = { + { NULL, sext, sext, sext, NULL, sext, sext, sext, s2f, s2f, s2f}, + { chop, NULL, sext, sext, chop, NULL, sext, sext, s2f, s2f, s2f}, + { chop, sexd, NULL, sext, chop, chop, NULL, sext, s2f, s2f, s2f}, + { chop, chop, chop, NULL, chop, chop, chop, NULL, s2f, s2f, s2f}, + { NULL, zext, zext, zext, NULL, zext, zext, zext, u2f, u2f, u2f}, + { chop, NULL, zext, zext, chop, NULL, zext, zext, u2f, u2f, u2f}, + { chop, chop, NULL, zext, chop, chop, NULL, zext, u2f, u2f, u2f}, + { chop, chop, chop, NULL, chop, chop, chop, NULL, u2f, u2f, u2f}, + { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , NULL,f2f, f2x}, + { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x, f2f, f2x}, + { d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x, d2x, d2d} +}; -ptx_reg_t chop(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - switch (to_width) { - case 8: - x.mask_and(0, 0xFF); - break; - case 16: - x.mask_and(0, 0xFFFF); - break; - case 32: - x.mask_and(0, 0xFFFFFFFF); - break; - case 64: +void ptx_round(ptx_reg_t& data, int rounding_mode, int type) +{ + if (rounding_mode == RN_OPTION) { + return; + } + switch ( rounding_mode ) { + case RZI_OPTION: + switch ( type ) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); assert(0); break; + case F16_TYPE: data.f16=truncf(data.f16);break;//assert(0); break; + case F32_TYPE: + data.f32 = truncf(data.f32); + break; + case F64_TYPE: + case FF64_TYPE: + if (data.f64 < 0) data.f64 = ceil(data.f64); //negative + else data.f64 = floor(data.f64); //positive + break; + default: assert(0); break; + } break; - default: - assert(0); - } - return x; -} - -ptx_reg_t sext(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - x = chop(x, 0, from_width, 0, rounding_mode, saturation_mode); - switch (from_width) { - case 8: - if (x.get_bit(7)) x.mask_or(0xFFFFFFFF, 0xFFFFFF00); - break; - case 16: - if (x.get_bit(15)) x.mask_or(0xFFFFFFFF, 0xFFFF0000); - break; - case 32: - if (x.get_bit(31)) x.mask_or(0xFFFFFFFF, 0x00000000); - break; - case 64: - break; - default: - assert(0); - } - return x; -} - -// sign extend depending on the destination register size - hack to get -// SobelFilter working in CUDA 4.2 -ptx_reg_t sexd(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - x = chop(x, 0, from_width, 0, rounding_mode, saturation_mode); - switch (to_width) { - case 8: - if (x.get_bit(7)) x.mask_or(0xFFFFFFFF, 0xFFFFFF00); - break; - case 16: - if (x.get_bit(15)) x.mask_or(0xFFFFFFFF, 0xFFFF0000); - break; - case 32: - if (x.get_bit(31)) x.mask_or(0xFFFFFFFF, 0x00000000); - break; - case 64: - break; - default: - assert(0); - } - return x; -} - -ptx_reg_t zext(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - return chop(x, 0, from_width, 0, rounding_mode, saturation_mode); -} - -int saturatei(int a, int max, int min) { - if (a > max) - a = max; - else if (a < min) - a = min; - return a; -} - -unsigned int saturatei(unsigned int a, unsigned int max) { - if (a > max) a = max; - return a; -} - -ptx_reg_t f2x(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - half mytemp; - half_float::half tmp_h; - // assert( from_width == 32); - - enum cudaRoundMode mode = cudaRoundZero; - switch (rounding_mode) { - case RZI_OPTION: - mode = cudaRoundZero; - break; - case RNI_OPTION: - mode = cudaRoundNearest; - break; - case RMI_OPTION: - mode = cudaRoundMinInf; - break; - case RPI_OPTION: - mode = cudaRoundPosInf; - break; - default: - break; - } - - ptx_reg_t y; - if (to_sign == 1) { // convert to 64-bit number first? - int tmp = cuda_math::float2int(x.f32, mode); - if ((x.u32 & 0x7f800000) == 0) tmp = 0; // round denorm. FP to 0 - if (saturation_mode && to_width < 32) { - tmp = saturatei(tmp, (1 << to_width) - 1, -(1 << to_width)); - } - switch (to_width) { - case 8: - y.s8 = (char)tmp; - break; - case 16: - y.s16 = (short)tmp; - break; - case 32: - y.s32 = (int)tmp; - break; - case 64: - y.s64 = (long long)tmp; - break; - default: - assert(0); - break; - } - } else if (to_sign == 0) { - unsigned int tmp = cuda_math::float2uint(x.f32, mode); - if ((x.u32 & 0x7f800000) == 0) tmp = 0; // round denorm. FP to 0 - if (saturation_mode && to_width < 32) { - tmp = saturatei(tmp, (1 << to_width) - 1); - } - switch (to_width) { - case 8: - y.u8 = (unsigned char)tmp; - break; - case 16: - y.u16 = (unsigned short)tmp; - break; - case 32: - y.u32 = (unsigned int)tmp; - break; - case 64: - y.u64 = (unsigned long long)tmp; - break; - default: - assert(0); - break; - } - } else { - switch (to_width) { - case 16: - y.f16 = half_float::half_cast::round_style>( - x.f32); // mytemp; - break; - case 32: - y.f32 = float(x.f16); - break; // handled by f2f - case 64: - y.f64 = x.f32; - break; - default: - assert(0); - break; - } - } - return y; -} - -double saturated2i(double a, double max, double min) { - if (a > max) - a = max; - else if (a < min) - a = min; - return a; -} - -ptx_reg_t d2x(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - assert(from_width == 64); - - double tmp; - switch (rounding_mode) { - case RZI_OPTION: - tmp = trunc(x.f64); - break; - case RNI_OPTION: - tmp = nearbyint(x.f64); - break; - case RMI_OPTION: - tmp = floor(x.f64); - break; - case RPI_OPTION: - tmp = ceil(x.f64); - break; - default: - tmp = x.f64; - break; - } - - ptx_reg_t y; - if (to_sign == 1) { - tmp = saturated2i(tmp, ((1 << (to_width - 1)) - 1), (1 << (to_width - 1))); - switch (to_width) { - case 8: - y.s8 = (char)tmp; - break; - case 16: - y.s16 = (short)tmp; - break; - case 32: - y.s32 = (int)tmp; - break; - case 64: - y.s64 = (long long)tmp; - break; - default: - assert(0); - break; - } - } else if (to_sign == 0) { - tmp = saturated2i(tmp, ((1 << (to_width - 1)) - 1), 0); - switch (to_width) { - case 8: - y.u8 = (unsigned char)tmp; - break; - case 16: - y.u16 = (unsigned short)tmp; - break; - case 32: - y.u32 = (unsigned int)tmp; - break; - case 64: - y.u64 = (unsigned long long)tmp; - break; - default: - assert(0); - break; - } - } else { - switch (to_width) { - case 16: - assert(0); - break; - case 32: - y.f32 = x.f64; - break; - case 64: - y.f64 = x.f64; // should be handled by d2d - break; - default: - assert(0); - break; - } - } - return y; -} - -ptx_reg_t s2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - ptx_reg_t y; - - if (from_width < 64) { // 32-bit conversion - y = sext(x, from_width, 32, 0, rounding_mode, saturation_mode); - - switch (to_width) { - case 16: - assert(0); - break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: - y.f32 = cuda_math::__int2float_rz(y.s32); - break; - case RN_OPTION: - y.f32 = cuda_math::__int2float_rn(y.s32); - break; - case RM_OPTION: - y.f32 = cuda_math::__int2float_rd(y.s32); - break; - case RP_OPTION: - y.f32 = cuda_math::__int2float_ru(y.s32); - break; - default: - break; - } - break; - case 64: - y.f64 = y.s32; - break; // no rounding needed - default: - assert(0); - break; - } - } else { - switch (to_width) { - case 16: - assert(0); - break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: - y.f32 = cuda_math::__ll2float_rz(y.s64); - break; - case RN_OPTION: - y.f32 = cuda_math::__ll2float_rn(y.s64); - break; - case RM_OPTION: - y.f32 = cuda_math::__ll2float_rd(y.s64); - break; - case RP_OPTION: - y.f32 = cuda_math::__ll2float_ru(y.s64); - break; - default: - break; - } - break; - case 64: - y.f64 = y.s64; - break; // no internal implementation found - default: - assert(0); - break; - } - } - - // saturating an integer to 1 or 0? - return y; -} - -ptx_reg_t u2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - ptx_reg_t y; - - if (from_width < 64) { // 32-bit conversion - y = zext(x, from_width, 32, 0, rounding_mode, saturation_mode); - - switch (to_width) { - case 16: - assert(0); - break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: - y.f32 = cuda_math::__uint2float_rz(y.u32); - break; - case RN_OPTION: - y.f32 = cuda_math::__uint2float_rn(y.u32); - break; - case RM_OPTION: - y.f32 = cuda_math::__uint2float_rd(y.u32); - break; - case RP_OPTION: - y.f32 = cuda_math::__uint2float_ru(y.u32); - break; - default: - break; - } - break; - case 64: - y.f64 = y.u32; - break; // no rounding needed - default: - assert(0); - break; - } - } else { - switch (to_width) { - case 16: - assert(0); - break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: - y.f32 = cuda_math::__ull2float_rn(y.u64); - break; - case RN_OPTION: - y.f32 = cuda_math::__ull2float_rn(y.u64); - break; - case RM_OPTION: - y.f32 = cuda_math::__ull2float_rn(y.u64); - break; - case RP_OPTION: - y.f32 = cuda_math::__ull2float_rn(y.u64); - break; - default: - break; - } - break; - case 64: - y.f64 = y.u64; - break; // no internal implementation found - default: - assert(0); - break; - } - } - - // saturating an integer to 1 or 0? - return y; -} - -ptx_reg_t f2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - ptx_reg_t y; - if (from_width == 16) { - half_float::detail::uint16 val = x.u16; - y.f32 = half_float::detail::half2float(val); - } else { - switch (rounding_mode) { - case RZI_OPTION: - y.f32 = truncf(x.f32); - break; - case RNI_OPTION: -#if CUDART_VERSION >= 3000 - y.f32 = nearbyintf(x.f32); -#else - y.f32 = cuda_math::__internal_nearbyintf(x.f32); -#endif - break; - case RMI_OPTION: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = floorf(x.f32); - } - break; - case RPI_OPTION: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = ceilf(x.f32); - } - break; - default: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = x.f32; - } - break; - } -#if CUDART_VERSION >= 3000 - if (isnanf(y.f32)) -#else - if (cuda_math::__cuda___isnanf(y.f32)) -#endif - { - y.u32 = 0x7fffffff; - } else if (saturation_mode) { - y.f32 = cuda_math::__saturatef(y.f32); - } - } - - return y; -} - -ptx_reg_t d2d(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - ptx_reg_t y; - switch (rounding_mode) { - case RZI_OPTION: - y.f64 = trunc(x.f64); - break; - case RNI_OPTION: -#if CUDART_VERSION >= 3000 - y.f64 = nearbyint(x.f64); -#else - y.f64 = cuda_math::__internal_nearbyintf(x.f64); -#endif - break; - case RMI_OPTION: - y.f64 = floor(x.f64); - break; - case RPI_OPTION: - y.f64 = ceil(x.f64); - break; - default: - y.f64 = x.f64; - break; - } - if (std::isnan(y.f64)) { - y.u64 = 0xfff8000000000000ull; - } else if (saturation_mode) { - y.f64 = cuda_math::__saturatef(y.f64); - } - return y; -} - -ptx_reg_t (*g_cvt_fn[11][11])(ptx_reg_t x, unsigned from_width, - unsigned to_width, int to_sign, int rounding_mode, - int saturation_mode) = { - {NULL, sext, sext, sext, NULL, sext, sext, sext, s2f, s2f, s2f}, - {chop, NULL, sext, sext, chop, NULL, sext, sext, s2f, s2f, s2f}, - {chop, sexd, NULL, sext, chop, chop, NULL, sext, s2f, s2f, s2f}, - {chop, chop, chop, NULL, chop, chop, chop, NULL, s2f, s2f, s2f}, - {NULL, zext, zext, zext, NULL, zext, zext, zext, u2f, u2f, u2f}, - {chop, NULL, zext, zext, chop, NULL, zext, zext, u2f, u2f, u2f}, - {chop, chop, NULL, zext, chop, chop, NULL, zext, u2f, u2f, u2f}, - {chop, chop, chop, NULL, chop, chop, chop, NULL, u2f, u2f, u2f}, - {f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, NULL, f2f, f2x}, - {f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2f, f2x}, - {d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2d}}; - -void ptx_round(ptx_reg_t &data, int rounding_mode, int type) { - if (rounding_mode == RN_OPTION) { - return; - } - switch (rounding_mode) { - case RZI_OPTION: - switch (type) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); - assert(0); - break; - case F16_TYPE: - data.f16 = truncf(data.f16); - break; // assert(0); break; - case F32_TYPE: - data.f32 = truncf(data.f32); - break; - case F64_TYPE: - case FF64_TYPE: - if (data.f64 < 0) - data.f64 = ceil(data.f64); // negative - else - data.f64 = floor(data.f64); // positive - break; - default: - assert(0); - break; - } - break; - case RNI_OPTION: - switch (type) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); - assert(0); - break; - case F16_TYPE: // assert(0); break; + case RNI_OPTION: + switch ( type ) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); assert(0); break; + case F16_TYPE:// assert(0); break; #if CUDART_VERSION >= 3000 - data.f16 = nearbyintf(data.f16); + data.f16 = nearbyintf(data.f16); #else - data.f16 = cuda_math::__cuda_nearbyintf(data.f16); + data.f16 = cuda_math::__cuda_nearbyintf(data.f16); #endif - break; - case F32_TYPE: + break; + case F32_TYPE: #if CUDART_VERSION >= 3000 - data.f32 = nearbyintf(data.f32); + data.f32 = nearbyintf(data.f32); #else - data.f32 = cuda_math::__cuda_nearbyintf(data.f32); + data.f32 = cuda_math::__cuda_nearbyintf(data.f32); #endif - break; - case F64_TYPE: - case FF64_TYPE: - data.f64 = round(data.f64); - break; - default: - assert(0); - break; - } - break; - case RMI_OPTION: - switch (type) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); - assert(0); - break; - case F16_TYPE: - data.f16 = floorf(data.f16); - break; // assert(0); break; - case F32_TYPE: - data.f32 = floorf(data.f32); - break; - case F64_TYPE: - case FF64_TYPE: - data.f64 = floor(data.f64); - break; - default: - assert(0); - break; + break; + case F64_TYPE: case FF64_TYPE: data.f64 = round(data.f64); break; + default: assert(0); break; } break; - case RPI_OPTION: - switch (type) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); - assert(0); - break; - case F16_TYPE: - data.f16 = ceilf(data.f16); - break; // assert(0); break; - case F32_TYPE: - data.f32 = ceilf(data.f32); - break; - case F64_TYPE: - case FF64_TYPE: - data.f64 = ceil(data.f64); - break; - default: - assert(0); - break; + case RMI_OPTION: + switch ( type ) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); assert(0); break; + case F16_TYPE: data.f16=floorf(data.f16);break;//assert(0); break; + case F32_TYPE: + data.f32 = floorf(data.f32); + break; + case F64_TYPE: case FF64_TYPE: data.f64 = floor(data.f64); break; + default: assert(0); break; } break; - default: - break; - } - - if (type == F32_TYPE) { -#if CUDART_VERSION >= 3000 - if (isnanf(data.f32)) -#else - if (cuda_math::__cuda___isnanf(data.f32)) -#endif - { - data.u32 = 0x7fffffff; - } - } - if ((type == F64_TYPE) || (type == FF64_TYPE)) { - if (std::isnan(data.f64)) { - data.u64 = 0xfff8000000000000ull; - } - } -} - -void ptx_saturate(ptx_reg_t &data, int saturation_mode, int type) { - if (!saturation_mode) { - return; - } - switch (type) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to clamp an integer to 1??\n"); - assert(0); - break; - case F16_TYPE: // assert(0); break; - if (data.f16 > 1.0f) data.f16 = 1.0f; // negative - if (data.f16 < 0.0f) data.f16 = 0.0f; // positive - break; - case F32_TYPE: - if (data.f32 > 1.0f) data.f32 = 1.0f; // negative - if (data.f32 < 0.0f) data.f32 = 0.0f; // positive - break; - case F64_TYPE: - case FF64_TYPE: - if (data.f64 > 1.0f) data.f64 = 1.0f; // negative - if (data.f64 < 0.0f) data.f64 = 0.0f; // positive - break; - default: - assert(0); - break; - } -} - -void cvt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned to_type = pI->get_type(); - unsigned from_type = pI->get_type2(); - unsigned rounding_mode = pI->rounding_mode(); - unsigned saturation_mode = pI->saturation_mode(); - - // if ( to_type == F16_TYPE || from_type == F16_TYPE ) - // abort(); - - int to_sign, from_sign; - size_t from_width, to_width; - unsigned src_fmt = - type_info_key::type_decode(from_type, from_width, from_sign); - unsigned dst_fmt = type_info_key::type_decode(to_type, to_width, to_sign); - - ptx_reg_t data = thread->get_operand_value(src1, dst, from_type, thread, 1); - - if (pI->is_neg()) { - switch (from_type) { - // Default to f32 for now, need to add support for others + case RPI_OPTION: + switch ( type ) { case S8_TYPE: - case U8_TYPE: - case B8_TYPE: - data.s8 = -data.s8; - break; case S16_TYPE: - case U16_TYPE: - case B16_TYPE: - data.s16 = -data.s16; - break; case S32_TYPE: - case U32_TYPE: - case B32_TYPE: - data.s32 = -data.s32; - break; case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: case U64_TYPE: - case B64_TYPE: - data.s64 = -data.s64; - break; - case F16_TYPE: - data.f16 = -data.f16; - break; - case F32_TYPE: - data.f32 = -data.f32; - break; - case F64_TYPE: - case FF64_TYPE: - data.f64 = -data.f64; - break; - default: - assert(0); - } - } - - if (g_cvt_fn[src_fmt][dst_fmt] != NULL) { - ptx_reg_t result = g_cvt_fn[src_fmt][dst_fmt]( - data, from_width, to_width, to_sign, rounding_mode, saturation_mode); - data = result; - } - - thread->set_operand_value(dst, data, to_type, thread, pI); -} - -void cvta_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t data; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - memory_space_t space = pI->get_space(); - bool to_non_generic = pI->is_to(); - - unsigned i_type = pI->get_type(); - ptx_reg_t from_addr = thread->get_operand_value(src1, dst, i_type, thread, 1); - addr_t from_addr_hw = (addr_t)from_addr.u64; - addr_t to_addr_hw = 0; - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); - - if (to_non_generic) { - switch (space.get_type()) { - case shared_space: - to_addr_hw = generic_to_shared(smid, from_addr_hw); - break; - case local_space: - to_addr_hw = generic_to_local(smid, hwtid, from_addr_hw); - break; - case global_space: - to_addr_hw = generic_to_global(from_addr_hw); - break; - default: - abort(); - } - } else { - switch (space.get_type()) { - case shared_space: - to_addr_hw = shared_to_generic(smid, from_addr_hw); - break; - case local_space: - to_addr_hw = local_to_generic(smid, hwtid, from_addr_hw) + - thread->get_local_mem_stack_pointer(); - break; // add stack ptr here so that it can be passed as a pointer at - // function call - case global_space: - to_addr_hw = global_to_generic(from_addr_hw); - break; - default: - abort(); - } - } - - ptx_reg_t to_addr; - to_addr.u64 = to_addr_hw; - thread->set_reg(dst.get_symbol(), to_addr); -} - -void div_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t data; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - - switch (i_type) { - case S8_TYPE: - data.s8 = src1_data.s8 / src2_data.s8; - break; - case S16_TYPE: - data.s16 = src1_data.s16 / src2_data.s16; - break; - case S32_TYPE: - data.s32 = src1_data.s32 / src2_data.s32; - break; - case S64_TYPE: - data.s64 = src1_data.s64 / src2_data.s64; - break; - case U8_TYPE: - data.u8 = src1_data.u8 / src2_data.u8; - break; - case U16_TYPE: - data.u16 = src1_data.u16 / src2_data.u16; - break; - case U32_TYPE: - data.u32 = src1_data.u32 / src2_data.u32; - break; - case U64_TYPE: - data.u64 = src1_data.u64 / src2_data.u64; - break; - case B8_TYPE: - data.u8 = src1_data.u8 / src2_data.u8; - break; - case B16_TYPE: - data.u16 = src1_data.u16 / src2_data.u16; - break; - case B32_TYPE: - data.u32 = src1_data.u32 / src2_data.u32; - break; - case B64_TYPE: - data.u64 = src1_data.u64 / src2_data.u64; - break; - case F16_TYPE: - data.f16 = src1_data.f16 / src2_data.f16; - break; // assert(0); break; - case F32_TYPE: - data.f32 = src1_data.f32 / src2_data.f32; - break; - case F64_TYPE: - case FF64_TYPE: - data.f64 = src1_data.f64 / src2_data.f64; - break; - default: - assert(0); - break; - } - thread->set_operand_value(dst, data, i_type, thread, pI); -} - -void dp4a_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - printf("DP4A instruction not implemented yet"); - assert(0); -} - -void ex2_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - - switch (i_type) { - case F32_TYPE: - data.f32 = cuda_math::__powf(2.0, src1_data.f32); - break; - default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); + printf("Trying to round an integer??\n"); assert(0); break; + case F16_TYPE: data.f16 = ceilf(data.f16); break; //assert(0); break; + case F32_TYPE: data.f32 = ceilf(data.f32); break; + case F64_TYPE: case FF64_TYPE: data.f64 = ceil(data.f64); break; + default: assert(0); break; + } break; - } + default: break; + } - thread->set_operand_value(dst, data, i_type, thread, pI); -} - -void exit_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - thread->set_done(); - thread->exitCore(); - thread->registerExit(); -} - -void mad_def(const ptx_instruction *pI, ptx_thread_info *thread, - bool use_carry = false); - -void fma_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - mad_def(pI, thread); -} - -void isspacep_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a; - bool t = false; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - memory_space_t space = pI->get_space(); - - a = thread->get_reg(src1.get_symbol()); - addr_t addr = (addr_t)a.u64; - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); - - switch (space.get_type()) { - case shared_space: - t = isspace_shared(smid, addr); - case local_space: - t = isspace_local(smid, hwtid, addr); - case global_space: - t = isspace_global(addr); - default: - abort(); - } - - ptx_reg_t p; - p.pred = t ? 1 : 0; - - thread->set_reg(dst.get_symbol(), p); -} - -void decode_space(memory_space_t &space, ptx_thread_info *thread, - const operand_info &op, memory_space *&mem, addr_t &addr) { - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); - - if (space == param_space_unclassified) { - // need to op to determine whether it refers to a kernel param or local - // param - const symbol *s = op.get_symbol(); - const type_info *t = s->type(); - type_info_key ti = t->get_key(); - if (ti.is_param_kernel()) - space = param_space_kernel; - else if (ti.is_param_local()) { - space = param_space_local; - } - // mov r1, param-label - else if (ti.is_reg()) { - space = param_space_kernel; - } else { - printf("GPGPU-Sim PTX: ERROR ** cannot resolve .param space for '%s'\n", - s->name().c_str()); - abort(); - } - } - switch (space.get_type()) { - case global_space: - mem = thread->get_global_memory(); - break; - case param_space_local: - case local_space: - mem = thread->m_local_mem; - addr += thread->get_local_mem_stack_pointer(); - break; - case tex_space: - mem = thread->get_tex_memory(); - break; - case surf_space: - mem = thread->get_surf_memory(); - break; - case param_space_kernel: - mem = thread->get_param_memory(); - break; - case shared_space: - mem = thread->m_shared_mem; - break; - case sstarr_space: - mem = thread->m_sstarr_mem; - break; - case const_space: - mem = thread->get_global_memory(); - break; - case generic_space: - if (thread->get_ptx_version().ver() >= 2.0) { - // convert generic address to memory space address - space = whichspace(addr); - switch (space.get_type()) { - case global_space: - mem = thread->get_global_memory(); - addr = generic_to_global(addr); - break; - case local_space: - mem = thread->m_local_mem; - addr = generic_to_local(smid, hwtid, addr); - break; - case shared_space: - mem = thread->m_shared_mem; - addr = generic_to_shared(smid, addr); - break; - default: - abort(); - } - } else { - abort(); + if (type == F32_TYPE) { +#if CUDART_VERSION >= 3000 + if (isnanf(data.f32)) +#else + if (cuda_math::__cuda___isnanf(data.f32)) +#endif + { + data.u32 = 0x7fffffff; } - break; - case param_space_unclassified: - case undefined_space: - default: - abort(); - } + } + if ((type == F64_TYPE)||(type == FF64_TYPE)) { + if (std::isnan(data.f64)) { + data.u64 = 0xfff8000000000000ull; + } + } } -void ld_exec(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned type = pI->get_type(); - - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, type, thread, 1); - ptx_reg_t data; - memory_space_t space = pI->get_space(); - unsigned vector_spec = pI->get_vector(); - - memory_space *mem = NULL; - addr_t addr = src1_data.u32; - - decode_space(space, thread, src1, mem, addr); - - size_t size; - int t; - data.u64 = 0; - type_info_key::type_decode(type, size, t); - if (!vector_spec) { - mem->read(addr, size / 8, &data.s64); - if (type == S16_TYPE || type == S32_TYPE) sign_extend(data, size, dst); - thread->set_operand_value(dst, data, type, thread, pI); - } else { - ptx_reg_t data1, data2, data3, data4; - mem->read(addr, size / 8, &data1.s64); - mem->read(addr + size / 8, size / 8, &data2.s64); - if (vector_spec != V2_TYPE) { // either V3 or V4 - mem->read(addr + 2 * size / 8, size / 8, &data3.s64); - if (vector_spec != V3_TYPE) { // v4 - mem->read(addr + 3 * size / 8, size / 8, &data4.s64); - thread->set_vector_operand_values(dst, data1, data2, data3, data4); - } else // v3 - thread->set_vector_operand_values(dst, data1, data2, data3, data3); - } else // v2 - thread->set_vector_operand_values(dst, data1, data2, data2, data2); - } - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; -} +void ptx_saturate(ptx_reg_t& data, int saturation_mode, int type) +{ + if (!saturation_mode) { + return; + } + switch ( type ) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to clamp an integer to 1??\n"); assert(0); break; + case F16_TYPE: //assert(0); break; + if (data.f16 > 1.0f) data.f16 = 1.0f; //negative + if (data.f16 < 0.0f) data.f16 = 0.0f; //positive + break; + case F32_TYPE: + if (data.f32 > 1.0f) data.f32 = 1.0f; //negative + if (data.f32 < 0.0f) data.f32 = 0.0f; //positive + break; + case F64_TYPE: + case FF64_TYPE: + if (data.f64 > 1.0f) data.f64 = 1.0f; //negative + if (data.f64 < 0.0f) data.f64 = 0.0f; //positive + break; + default: assert(0); break; + } + +} + +void cvt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned to_type = pI->get_type(); + unsigned from_type = pI->get_type2(); + unsigned rounding_mode = pI->rounding_mode(); + unsigned saturation_mode = pI->saturation_mode(); + +// if ( to_type == F16_TYPE || from_type == F16_TYPE ) +// abort(); + + int to_sign, from_sign; + size_t from_width, to_width; + unsigned src_fmt = type_info_key::type_decode(from_type, from_width, from_sign); + unsigned dst_fmt = type_info_key::type_decode(to_type, to_width, to_sign); + + ptx_reg_t data = thread->get_operand_value(src1, dst, from_type, thread, 1); + + if(pI->is_neg()){ + + switch( from_type ) { + // Default to f32 for now, need to add support for others + case S8_TYPE: + case U8_TYPE: + case B8_TYPE: + data.s8 = -data.s8; + break; + case S16_TYPE: + case U16_TYPE: + case B16_TYPE: + data.s16 = -data.s16; + break; + case S32_TYPE: + case U32_TYPE: + case B32_TYPE: + data.s32 = -data.s32; + break; + case S64_TYPE: + case U64_TYPE: + case B64_TYPE: + data.s64 = -data.s64; + break; + case F16_TYPE: + data.f16 = -data.f16; + break; + case F32_TYPE: + data.f32 = -data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = -data.f64; + break; + default: + assert(0); + } -void ld_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ld_exec(pI, thread); -} -void ldu_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ld_exec(pI, thread); -} + } -void mma_st_impl(const ptx_instruction *pI, core_t *core, warp_inst_t &inst) { - size_t size; - unsigned smid; - int t; - int thrd, k; - ptx_thread_info *thread; - - const operand_info &src = pI->operand_lookup(1); - const operand_info &src1 = pI->operand_lookup(0); - const operand_info &src2 = pI->operand_lookup(2); - int tid; - unsigned type = pI->get_type(); - unsigned wmma_type = pI->get_wmma_type(); - unsigned wmma_layout = pI->get_wmma_layout(0); - int stride; - - if (core->get_gpu()->is_functional_sim()) - tid = inst.warp_id_func() * core->get_warp_size(); - else - tid = inst.warp_id() * core->get_warp_size(); - - _memory_op_t insn_memory_op = - pI->has_memory_read() ? memory_load : memory_store; - for (thrd = 0; thrd < core->get_warp_size(); thrd++) { - thread = core->get_thread_info()[tid + thrd]; - ptx_reg_t addr_reg = thread->get_operand_value(src1, src, type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, src, type, thread, 1); - const operand_info &src_a = pI->operand_lookup(1); - unsigned nelem = src_a.get_vect_nelem(); - ptx_reg_t *v = new ptx_reg_t[8]; - thread->get_vector_operand_values(src_a, v, nelem); - stride = src2_data.u32; - memory_space_t space = pI->get_space(); + if ( g_cvt_fn[src_fmt][dst_fmt] != NULL ) { + ptx_reg_t result = g_cvt_fn[src_fmt][dst_fmt](data,from_width,to_width,to_sign, rounding_mode, saturation_mode); + data = result; + } - memory_space *mem = NULL; - addr_t addr = addr_reg.u32; + thread->set_operand_value(dst, data, to_type, thread, pI ); +} - new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; - int num_mem_txn = 0; +void cvta_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t data; - smid = thread->get_hw_sid(); - if (whichspace(addr) == shared_space) { - addr = generic_to_shared(smid, addr); - space = shared_space; - } - decode_space(space, thread, src1, mem, addr); - - type_info_key::type_decode(type, size, t); - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma_st: thrd=%d, addr=%x, fp(size=%zu), stride=%d\n", thrd, - addr_reg.u32, size, src2_data.u32); - addr_t new_addr = - addr + - thread_group_offset(thrd, wmma_type, wmma_layout, type, stride) * size / - 8; - addr_t push_addr; - - ptx_reg_t nw_v[8]; - for (k = 0; k < 8; k++) { - if (k % 2 == 0) - nw_v[k].s64 = (v[k / 2].s64 & 0xffff); - else - nw_v[k].s64 = ((v[k / 2].s64 & 0xffff0000) >> 16); - } + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + memory_space_t space = pI->get_space(); + bool to_non_generic = pI->is_to(); - for (k = 0; k < 8; k++) { - if (type == F32_TYPE) { - // mem->write(new_addr+4*acc_float_offset(k,wmma_layout,stride),size/8,&v[k].s64,thread,pI); - push_addr = new_addr + 4 * acc_float_offset(k, wmma_layout, stride); - mem->write(push_addr, size / 8, &v[k].s64, thread, pI); - mem_txn_addr[num_mem_txn++] = push_addr; - - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - printf( - "wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n", - thrd, v[0].s64, v[1].s64, v[2].s64, v[3].s64, v[4].s64, v[5].s64, - v[6].s64, v[7].s64); - float temp; - int l; - printf("thread=%d:", thrd); - for (l = 0; l < 8; l++) { - temp = v[l].f32; - printf("%.2f", temp); - } - printf("\n"); - } - } else if (type == F16_TYPE) { - if (wmma_layout == ROW) { - // mem->write(new_addr+k*2,size/8,&nw_v[k].s64,thread,pI); - push_addr = new_addr + k * 2; - mem->write(push_addr, size / 8, &nw_v[k].s64, thread, pI); - if (k % 2 == 0) mem_txn_addr[num_mem_txn++] = push_addr; - } else if (wmma_layout == COL) { - // mem->write(new_addr+k*2*stride,size/8,&nw_v[k].s64,thread,pI); - push_addr = new_addr + k * 2 * stride; - mem->write(push_addr, size / 8, &nw_v[k].s64, thread, pI); - mem_txn_addr[num_mem_txn++] = push_addr; - } + unsigned i_type = pI->get_type(); + ptx_reg_t from_addr = thread->get_operand_value(src1,dst,i_type,thread,1); + addr_t from_addr_hw = (addr_t)from_addr.u64; + addr_t to_addr_hw = 0; + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf( - "wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n", - thrd, nw_v[0].s64, nw_v[1].s64, nw_v[2].s64, nw_v[3].s64, - nw_v[4].s64, nw_v[5].s64, nw_v[6].s64, nw_v[7].s64); + if( to_non_generic ) { + switch( space.get_type() ) { + case shared_space: to_addr_hw = generic_to_shared( smid, from_addr_hw ); break; + case local_space: to_addr_hw = generic_to_local( smid, hwtid, from_addr_hw ); break; + case global_space: to_addr_hw = generic_to_global(from_addr_hw ); break; + default: abort(); } - } - - delete[] v; - inst.space = space; - inst.set_addr(thrd, (new_addr_type *)mem_txn_addr, num_mem_txn); - - if ((type == F16_TYPE) && - (wmma_layout == COL)) // check the profiling xls for details - inst.data_size = 2; // 2 byte transaction - else - inst.data_size = 4; // 4 byte transaction + } else { + switch( space.get_type() ) { + case shared_space: to_addr_hw = shared_to_generic( smid, from_addr_hw ); break; + case local_space: to_addr_hw = local_to_generic( smid, hwtid, from_addr_hw ) + + thread->get_local_mem_stack_pointer(); break; // add stack ptr here so that it can be passed as a pointer at function call + case global_space: to_addr_hw = global_to_generic( from_addr_hw ); break; + default: abort(); + } + } + + ptx_reg_t to_addr; + to_addr.u64 = to_addr_hw; + thread->set_reg(dst.get_symbol(),to_addr); +} + +void div_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + + switch ( i_type ) { + case S8_TYPE: + data.s8 = src1_data.s8 / src2_data.s8 ; break; + case S16_TYPE: + data.s16 = src1_data.s16 / src2_data.s16; break; + case S32_TYPE: + data.s32 = src1_data.s32 / src2_data.s32; break; + case S64_TYPE: + data.s64 = src1_data.s64 / src2_data.s64; break; + case U8_TYPE: + data.u8 = src1_data.u8 / src2_data.u8 ; break; + case U16_TYPE: + data.u16 = src1_data.u16 / src2_data.u16; break; + case U32_TYPE: + data.u32 = src1_data.u32 / src2_data.u32; break; + case U64_TYPE: + data.u64 = src1_data.u64 / src2_data.u64; break; + case B8_TYPE: + data.u8 = src1_data.u8 / src2_data.u8 ; break; + case B16_TYPE: + data.u16 = src1_data.u16 / src2_data.u16; break; + case B32_TYPE: + data.u32 = src1_data.u32 / src2_data.u32; break; + case B64_TYPE: + data.u64 = src1_data.u64 / src2_data.u64; break; + case F16_TYPE: data.f16 = src1_data.f16 / src2_data.f16; break;//assert(0); break; + case F32_TYPE: data.f32 = src1_data.f32 / src2_data.f32; break; + case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 / src2_data.f64; break; + default: assert(0); break; + } + thread->set_operand_value(dst,data, i_type, thread,pI); +} + +void dp4a_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + printf("DP4A instruction not implemented yet"); + assert(0); + +} + +void ex2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + + switch ( i_type ) { + case F32_TYPE: + data.f32 = cuda_math::__powf(2.0, src1_data.f32); + break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst,data, i_type, thread,pI); +} - assert(inst.memory_op == insn_memory_op); - // thread->m_last_effective_address = addr; - // thread->m_last_memory_space = space; - } +void exit_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + thread->set_done(); + thread->exitCore(); + thread->registerExit(); } -void mma_ld_impl(const ptx_instruction *pI, core_t *core, warp_inst_t &inst) { - size_t size; - int t, i; - unsigned smid; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned type = pI->get_type(); - unsigned wmma_type = pI->get_wmma_type(); - unsigned wmma_layout = pI->get_wmma_layout(0); - int tid; - int thrd, stride; - ptx_thread_info *thread; - - if (core->get_gpu()->is_functional_sim()) - tid = inst.warp_id_func() * core->get_warp_size(); - else - tid = inst.warp_id() * core->get_warp_size(); - - _memory_op_t insn_memory_op = - pI->has_memory_read() ? memory_load : memory_store; - - for (thrd = 0; thrd < core->get_warp_size(); thrd++) { - thread = core->get_thread_info()[tid + thrd]; - ptx_reg_t src1_data = - thread->get_operand_value(src1, dst, U32_TYPE, thread, 1); - ptx_reg_t src2_data = - thread->get_operand_value(src2, dst, U32_TYPE, thread, 1); - stride = src2_data.u32; - memory_space_t space = pI->get_space(); - - memory_space *mem = NULL; - addr_t addr = src1_data.u32; - smid = thread->get_hw_sid(); - if (whichspace(addr) == shared_space) { - addr = generic_to_shared(smid, addr); - space = shared_space; - } +void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry = false ); - decode_space(space, thread, src1, mem, addr); - type_info_key::type_decode(type, size, t); - - ptx_reg_t data[16]; - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma_ld: thrd=%d,addr=%x, fpsize=%zu, stride=%d\n", thrd, - src1_data.u32, size, src2_data.u32); - - addr_t new_addr = - addr + - thread_group_offset(thrd, wmma_type, wmma_layout, type, stride) * size / - 8; - addr_t fetch_addr; - new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; - int num_mem_txn = 0; - - if (wmma_type == LOAD_A) { - for (i = 0; i < 16; i++) { - if (wmma_layout == ROW) { - // mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr = new_addr + 2 * i; - mem->read(fetch_addr, size / 8, &data[i].s64); - } else if (wmma_layout == COL) { - // mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); - fetch_addr = new_addr + 2 * (i % 4) + 2 * stride * 4 * (i / 4); - mem->read(fetch_addr, size / 8, &data[i].s64); - } else { - printf("mma_ld:wrong_layout_type\n"); - abort(); - } - if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; - } - } else if (wmma_type == LOAD_B) { - for (i = 0; i < 16; i++) { - if (wmma_layout == COL) { - // mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr = new_addr + 2 * i; - mem->read(fetch_addr, size / 8, &data[i].s64); - } else if (wmma_layout == ROW) { - // mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); - fetch_addr = new_addr + 2 * (i % 4) + 2 * stride * 4 * (i / 4); - mem->read(fetch_addr, size / 8, &data[i].s64); - } else { - printf("mma_ld:wrong_layout_type\n"); - abort(); - } - if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; - } - } else if (wmma_type == LOAD_C) { - for (i = 0; i < 8; i++) { - if (type == F16_TYPE) { - if (wmma_layout == ROW) { - // mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr = new_addr + 2 * i; - mem->read(fetch_addr, size / 8, &data[i].s64); - if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; - } else if (wmma_layout == COL) { - // mem->read(new_addr+2*stride*i,size/8,&data[i].s64); - fetch_addr = new_addr + 2 * stride * i; - mem->read(fetch_addr, size / 8, &data[i].s64); - mem_txn_addr[num_mem_txn++] = fetch_addr; - } else { - printf("mma_ld:wrong_type\n"); - abort(); - } - } else if (type == F32_TYPE) { - // mem->read(new_addr+4*acc_float_offset(i,wmma_layout,stride),size/8,&data[i].s64); - fetch_addr = new_addr + 4 * acc_float_offset(i, wmma_layout, stride); - mem->read(fetch_addr, size / 8, &data[i].s64); - mem_txn_addr[num_mem_txn++] = fetch_addr; - } else { - printf("wrong type"); - abort(); - } - } - } else { - printf("wrong wmma type\n"); - ; - abort(); - } - // generate timing memory request - inst.space = space; - inst.set_addr(thrd, (new_addr_type *)mem_txn_addr, num_mem_txn); - - if ((wmma_type == LOAD_C) && (type == F16_TYPE) && - (wmma_layout == COL)) // memory address is scattered, check the - // profiling xls for more detail. - inst.data_size = 2; // 2 byte transaction - else - inst.data_size = 4; // 4 byte transaction - assert(inst.memory_op == insn_memory_op); - - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - if (type == F16_TYPE) { - printf("\nmma_ld:thread%d= ", thrd); - for (i = 0; i < 16; i++) { - printf("%llx ", data[i].u64); - } - printf("\n"); +void fma_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + mad_def(pI,thread); +} - printf("\nmma_ld:thread%d= ", thrd); - float temp; - for (i = 0; i < 16; i++) { - temp = data[i].f16; - printf("%.2f ", temp); - } - printf("\n"); - } else { - printf("\nmma_ld:thread%d= ", thrd); - for (i = 0; i < 8; i++) { - printf("%.2f ", data[i].f32); - } - printf("\n"); - printf("\nmma_ld:thread%d= ", thrd); - for (i = 0; i < 8; i++) { - printf("%llx ", data[i].u64); - } - printf("\n"); - } - } +void isspacep_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a; + bool t=false; - if ((wmma_type == LOAD_C) && (type == F32_TYPE)) { - thread->set_wmma_vector_operand_values(dst, data[0], data[1], data[2], - data[3], data[4], data[5], data[6], - data[7]); - } else { - ptx_reg_t nw_data[8]; - int num_reg; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + memory_space_t space = pI->get_space(); - if (wmma_type == LOAD_C) - num_reg = 4; - else - num_reg = 8; + a = thread->get_reg(src1.get_symbol()); + addr_t addr = (addr_t)a.u64; + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); - for (i = 0; i < num_reg; i++) { - nw_data[i].s64 = ((data[2 * i].s64 & 0xffff) << 16) | - ((data[2 * i + 1].s64 & 0xffff)); - } + switch( space.get_type() ) { + case shared_space: t = isspace_shared( smid, addr ); + case local_space: t = isspace_local( smid, hwtid, addr ); + case global_space: t = isspace_global( addr ); + default: abort(); + } - if (wmma_type == LOAD_C) - thread->set_vector_operand_values(dst, nw_data[0], nw_data[1], - nw_data[2], nw_data[3]); - else - thread->set_wmma_vector_operand_values( - dst, nw_data[0], nw_data[1], nw_data[2], nw_data[3], nw_data[4], - nw_data[5], nw_data[6], nw_data[7]); - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - printf( - "mma_ld:data[0].s64=%llx,data[1].s64=%llx,new_data[0].s64=%llx\n", - data[0].u64, data[1].u64, nw_data[0].u64); - printf( - "mma_ld:data[2].s64=%llx,data[3].s64=%llx,new_data[1].s64=%llx\n", - data[2].u64, data[3].u64, nw_data[1].u64); - printf( - "mma_ld:data[4].s64=%llx,data[5].s64=%llx,new_data[2].s64=%llx\n", - data[4].u64, data[5].u64, nw_data[2].u64); - printf( - "mma_ld:data[6].s64=%llx,data[7].s64=%llx,new_data[3].s64=%llx\n", - data[6].u64, data[7].u64, nw_data[3].u64); - if (wmma_type != LOAD_C) { - printf( - "mma_ld:data[8].s64=%llx,data[9].s64=%llx,new_data[4].s64=%llx\n", - data[8].u64, data[9].u64, nw_data[4].s64); - printf( - "mma_ld:data[10].s64=%llx,data[11].s64=%llx,new_data[5].s64=%" - "llx\n", - data[10].u64, data[11].u64, nw_data[5].u64); - printf( - "mma_ld:data[12].s64=%llx,data[13].s64=%llx,new_data[6].s64=%" - "llx\n", - data[12].u64, data[13].u64, nw_data[6].u64); - printf( - "mma_ld:data[14].s64=%llx,data[15].s64=%llx,new_data[7].s64=%" - "llx\n", - data[14].u64, data[15].u64, nw_data[3].u64); - } - } - } + ptx_reg_t p; + p.pred = t?1:0; - // thread->m_last_effective_address = addr; - // thread->m_last_memory_space = space; - } + thread->set_reg(dst.get_symbol(),p); } -void lg2_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - - a = thread->get_operand_value(src1, dst, i_type, thread, 1); +void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand_info &op, memory_space *&mem, addr_t &addr) +{ + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); - switch (i_type) { - case F32_TYPE: - d.f32 = log(a.f32) / log(2); + if( space == param_space_unclassified ) { + // need to op to determine whether it refers to a kernel param or local param + const symbol *s = op.get_symbol(); + const type_info *t = s->type(); + type_info_key ti = t->get_key(); + if( ti.is_param_kernel() ) + space = param_space_kernel; + else if( ti.is_param_local() ) { + space = param_space_local; + } + //mov r1, param-label + else if (ti.is_reg() ){ + space = param_space_kernel; + } + else { + printf("GPGPU-Sim PTX: ERROR ** cannot resolve .param space for '%s'\n", s->name().c_str() ); + abort(); + } + } + switch ( space.get_type() ) { + case global_space: mem = thread->get_global_memory(); break; + case param_space_local: + case local_space: + mem = thread->m_local_mem; + addr += thread->get_local_mem_stack_pointer(); + break; + case tex_space: mem = thread->get_tex_memory(); break; + case surf_space: mem = thread->get_surf_memory(); break; + case param_space_kernel: mem = thread->get_param_memory(); break; + case shared_space: mem = thread->m_shared_mem; break; + case sstarr_space: mem = thread->m_sstarr_mem; break; + case const_space: mem = thread->get_global_memory(); break; + case generic_space: + if( thread->get_ptx_version().ver() >= 2.0 ) { + // convert generic address to memory space address + space = whichspace(addr); + switch ( space.get_type() ) { + case global_space: mem = thread->get_global_memory(); addr = generic_to_global(addr); break; + case local_space: mem = thread->m_local_mem; addr = generic_to_local(smid,hwtid,addr); break; + case shared_space: mem = thread->m_shared_mem; addr = generic_to_shared(smid,addr); break; + default: abort(); + } + } else { + abort(); + } break; - default: + case param_space_unclassified: + case undefined_space: + default: + abort(); + } +} + +void ld_exec( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned type = pI->get_type(); + + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, type, thread, 1); + ptx_reg_t data; + memory_space_t space = pI->get_space(); + unsigned vector_spec = pI->get_vector(); + + memory_space *mem = NULL; + addr_t addr = src1_data.u32; + + decode_space(space,thread,src1,mem,addr); + + size_t size; + int t; + data.u64=0; + type_info_key::type_decode(type,size,t); + if (!vector_spec) { + mem->read(addr,size/8,&data.s64); + if( type == S16_TYPE || type == S32_TYPE ) + sign_extend(data,size,dst); + thread->set_operand_value(dst,data, type, thread, pI); + } else { + ptx_reg_t data1, data2, data3, data4; + mem->read(addr,size/8,&data1.s64); + mem->read(addr+size/8,size/8,&data2.s64); + if (vector_spec != V2_TYPE) { //either V3 or V4 + mem->read(addr+2*size/8,size/8,&data3.s64); + if (vector_spec != V3_TYPE) { //v4 + mem->read(addr+3*size/8,size/8,&data4.s64); + thread->set_vector_operand_values(dst,data1,data2,data3,data4); + } else //v3 + thread->set_vector_operand_values(dst,data1,data2,data3,data3); + } else //v2 + thread->set_vector_operand_values(dst,data1,data2,data2,data2); + } + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; +} + +void ld_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ld_exec(pI,thread); +} +void ldu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ld_exec(pI,thread); +} + +void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t &inst ) +{ + size_t size; + unsigned smid; + int t; + int thrd, k; + ptx_thread_info *thread; + + const operand_info &src = pI->operand_lookup(1); + const operand_info &src1 = pI->operand_lookup(0); + const operand_info &src2 = pI->operand_lookup(2); + int tid ; + unsigned type = pI->get_type(); + unsigned wmma_type = pI->get_wmma_type(); + unsigned wmma_layout = pI->get_wmma_layout(0); + int stride; + + if(core->get_gpu()->is_functional_sim()) + tid= inst.warp_id_func()*core->get_warp_size(); + else + tid= inst.warp_id()*core->get_warp_size(); + + _memory_op_t insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; + for (thrd=0; thrd < core->get_warp_size(); thrd++) { + thread = core->get_thread_info()[tid+thrd]; + ptx_reg_t addr_reg = thread->get_operand_value(src1, src, type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, src, type, thread, 1); + const operand_info &src_a= pI->operand_lookup(1); + unsigned nelem = src_a.get_vect_nelem(); + ptx_reg_t* v= new ptx_reg_t[8]; + thread->get_vector_operand_values( src_a, v, nelem ); + stride = src2_data.u32; + + memory_space_t space = pI->get_space(); + + memory_space *mem = NULL; + addr_t addr = addr_reg.u32; + + new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; + int num_mem_txn=0; + + smid = thread->get_hw_sid(); + if( whichspace(addr) == shared_space ) { + addr= generic_to_shared(smid,addr); + space = shared_space; + } + decode_space(space,thread,src1,mem,addr); + + type_info_key::type_decode(type, size, t); + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma_st: thrd=%d, addr=%x, fp(size=%zu), stride=%d\n", thrd, addr_reg.u32, size, src2_data.u32); + addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8; + addr_t push_addr; + + ptx_reg_t nw_v[8]; + for(k=0;k<8;k++){ + if(k%2==0) + nw_v[k].s64=(v[k/2].s64&0xffff); + else + nw_v[k].s64=((v[k/2].s64&0xffff0000)>>16); + } + + for(k=0;k<8;k++){ + if(type==F32_TYPE){ + //mem->write(new_addr+4*acc_float_offset(k,wmma_layout,stride),size/8,&v[k].s64,thread,pI); + push_addr=new_addr+4*acc_float_offset(k,wmma_layout,stride); + mem->write(push_addr,size/8,&v[k].s64,thread,pI); + mem_txn_addr[num_mem_txn++]=push_addr; + + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ + printf("wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n",thrd,v[0].s64,v[1].s64,v[2].s64,v[3].s64,v[4].s64,v[5].s64,v[6].s64,v[7].s64); + float temp; + int l; + printf("thread=%d:",thrd); + for(l=0;l<8;l++){ + temp=v[l].f32; + printf("%.2f",temp); + } + printf("\n"); + } + } + else if(type==F16_TYPE){ + if(wmma_layout==ROW){ + //mem->write(new_addr+k*2,size/8,&nw_v[k].s64,thread,pI); + push_addr=new_addr+k*2; + mem->write(push_addr,size/8,&nw_v[k].s64,thread,pI); + if(k%2==0) + mem_txn_addr[num_mem_txn++]=push_addr; + } + else if(wmma_layout==COL){ + //mem->write(new_addr+k*2*stride,size/8,&nw_v[k].s64,thread,pI); + push_addr=new_addr+k*2*stride; + mem->write(push_addr,size/8,&nw_v[k].s64,thread,pI); + mem_txn_addr[num_mem_txn++]=push_addr; + } + + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n",thrd,nw_v[0].s64,nw_v[1].s64,nw_v[2].s64,nw_v[3].s64,nw_v[4].s64,nw_v[5].s64,nw_v[6].s64,nw_v[7].s64); + } + } + + delete [] v; + inst.space = space; + inst.set_addr(thrd, (new_addr_type *)mem_txn_addr , num_mem_txn); + + if((type==F16_TYPE)&&(wmma_layout==COL))//check the profiling xls for details + inst.data_size = 2; // 2 byte transaction + else + inst.data_size = 4; // 4 byte transaction + + assert( inst.memory_op == insn_memory_op ); + //thread->m_last_effective_address = addr; + //thread->m_last_memory_space = space; + } +} + +void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t &inst ) +{ + size_t size; + int t,i; + unsigned smid; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned type = pI->get_type(); + unsigned wmma_type = pI->get_wmma_type(); + unsigned wmma_layout = pI->get_wmma_layout(0); + int tid; + int thrd,stride; + ptx_thread_info *thread; + + + if(core->get_gpu()->is_functional_sim()) + tid= inst.warp_id_func()*core->get_warp_size(); + else + tid= inst.warp_id()*core->get_warp_size(); + + _memory_op_t insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; + + for (thrd=0; thrd < core->get_warp_size(); thrd++){ + thread = core->get_thread_info()[tid+thrd]; + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, U32_TYPE, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, dst, U32_TYPE, thread, 1); + stride=src2_data.u32; + memory_space_t space = pI->get_space(); + + memory_space *mem = NULL; + addr_t addr = src1_data.u32; + smid = thread->get_hw_sid(); + if( whichspace(addr) == shared_space ) { + addr= generic_to_shared(smid,addr); + space = shared_space; + } + + decode_space(space,thread,src1,mem,addr); + type_info_key::type_decode(type, size, t); + + ptx_reg_t data[16]; + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma_ld: thrd=%d,addr=%x, fpsize=%zu, stride=%d\n", thrd, src1_data.u32, size, src2_data.u32); + + addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8; + addr_t fetch_addr; + new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; + int num_mem_txn=0; + + if(wmma_type==LOAD_A){ + for(i=0;i<16;i++){ + if(wmma_layout==ROW){ + //mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr=new_addr+2*i; + mem->read(fetch_addr,size/8,&data[i].s64); + } + else if(wmma_layout==COL){ + //mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); + fetch_addr=new_addr+2*(i%4)+2*stride*4*(i/4); + mem->read(fetch_addr,size/8,&data[i].s64); + } + else{ + printf("mma_ld:wrong_layout_type\n"); + abort(); + + } + if(i%2==0) + mem_txn_addr[num_mem_txn++]=fetch_addr; + } + } + else if(wmma_type==LOAD_B){ + for(i=0;i<16;i++){ + if(wmma_layout==COL){ + //mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr=new_addr+2*i; + mem->read(fetch_addr,size/8,&data[i].s64); + } + else if(wmma_layout==ROW){ + //mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); + fetch_addr=new_addr+2*(i%4)+2*stride*4*(i/4); + mem->read(fetch_addr,size/8,&data[i].s64); + } + else{ + printf("mma_ld:wrong_layout_type\n"); + abort(); + } + if(i%2==0) + mem_txn_addr[num_mem_txn++]=fetch_addr; + } + } + else if(wmma_type==LOAD_C){ + for(i=0;i<8;i++){ + if(type==F16_TYPE){ + if(wmma_layout==ROW){ + //mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr=new_addr+2*i; + mem->read(fetch_addr,size/8,&data[i].s64); + if(i%2==0) + mem_txn_addr[num_mem_txn++]=fetch_addr; + } + else if(wmma_layout==COL){ + //mem->read(new_addr+2*stride*i,size/8,&data[i].s64); + fetch_addr=new_addr+2*stride*i; + mem->read(fetch_addr,size/8,&data[i].s64); + mem_txn_addr[num_mem_txn++]=fetch_addr; + } + else{ + printf("mma_ld:wrong_type\n"); + abort(); + } + } + else if(type==F32_TYPE){ + //mem->read(new_addr+4*acc_float_offset(i,wmma_layout,stride),size/8,&data[i].s64); + fetch_addr=new_addr+4*acc_float_offset(i,wmma_layout,stride); + mem->read(fetch_addr,size/8,&data[i].s64); + mem_txn_addr[num_mem_txn++]=fetch_addr; + } + else{ + printf("wrong type"); + abort(); + } + } + } + else{ + printf("wrong wmma type\n");; + abort(); + } + //generate timing memory request + inst.space = space; + inst.set_addr(thrd, (new_addr_type *)mem_txn_addr , num_mem_txn); + + if((wmma_type==LOAD_C)&&(type==F16_TYPE)&&(wmma_layout==COL))//memory address is scattered, check the profiling xls for more detail. + inst.data_size = 2; // 2 byte transaction + else + inst.data_size = 4; // 4 byte transaction + assert( inst.memory_op == insn_memory_op ); + + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ + if(type==F16_TYPE){ + printf("\nmma_ld:thread%d= ",thrd); + for(i=0;i<16;i++){ + printf("%llx ",data[i].u64); + } + printf("\n"); + + printf("\nmma_ld:thread%d= ",thrd); + float temp; + for(i=0;i<16;i++){ + temp=data[i].f16; + printf("%.2f ",temp); + } + printf("\n"); + } + else{ + printf("\nmma_ld:thread%d= ",thrd); + for(i=0;i<8;i++){ + printf("%.2f ",data[i].f32); + } + printf("\n"); + printf("\nmma_ld:thread%d= ",thrd); + for(i=0;i<8;i++){ + printf("%llx ",data[i].u64); + } + printf("\n"); + } + } + + if((wmma_type==LOAD_C)&&(type==F32_TYPE)){ + thread->set_wmma_vector_operand_values(dst,data[0],data[1],data[2],data[3],data[4],data[5],data[6],data[7]); + } + else{ + ptx_reg_t nw_data[8]; + int num_reg; + + if(wmma_type==LOAD_C) + num_reg=4; + else + num_reg=8; + + for(i=0;iset_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3]); + else + thread->set_wmma_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3],nw_data[4],nw_data[5],nw_data[6],nw_data[7]); + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ + printf("mma_ld:data[0].s64=%llx,data[1].s64=%llx,new_data[0].s64=%llx\n",data[0].u64,data[1].u64,nw_data[0].u64); + printf("mma_ld:data[2].s64=%llx,data[3].s64=%llx,new_data[1].s64=%llx\n",data[2].u64,data[3].u64,nw_data[1].u64); + printf("mma_ld:data[4].s64=%llx,data[5].s64=%llx,new_data[2].s64=%llx\n",data[4].u64,data[5].u64,nw_data[2].u64); + printf("mma_ld:data[6].s64=%llx,data[7].s64=%llx,new_data[3].s64=%llx\n",data[6].u64,data[7].u64,nw_data[3].u64); + if(wmma_type!=LOAD_C){ + printf("mma_ld:data[8].s64=%llx,data[9].s64=%llx,new_data[4].s64=%llx\n",data[8].u64,data[9].u64,nw_data[4].s64); + printf("mma_ld:data[10].s64=%llx,data[11].s64=%llx,new_data[5].s64=%llx\n",data[10].u64,data[11].u64,nw_data[5].u64); + printf("mma_ld:data[12].s64=%llx,data[13].s64=%llx,new_data[6].s64=%llx\n",data[12].u64,data[13].u64,nw_data[6].u64); + printf("mma_ld:data[14].s64=%llx,data[15].s64=%llx,new_data[7].s64=%llx\n",data[14].u64,data[15].u64,nw_data[3].u64); + } + } + } + + //thread->m_last_effective_address = addr; + //thread->m_last_memory_space = space; + } +} + +void lg2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + + switch ( i_type ) { + case F32_TYPE: + d.f32 = log(a.f32)/log(2); + break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -void mad24_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t d, t; +void mad24_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t d, t; - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - unsigned sat_mode = pI->saturation_mode(); + unsigned sat_mode = pI->saturation_mode(); - assert(!pI->is_wide()); + assert( !pI->is_wide() ); - switch (i_type) { - case S32_TYPE: + switch ( i_type ) { + case S32_TYPE: t.s64 = a.s32 * b.s32; - if (pI->is_hi()) { - d.s64 = (t.s64 >> 16) + c.s32; - if (sat_mode) { - if (d.s64 > (int)0x7FFFFFFF) - d.s64 = (int)0x7FFFFFFF; - else if (d.s64 < (int)0x80000000) - d.s64 = (int)0x80000000; - } - } else if (pI->is_lo()) - d.s64 = t.s32 + c.s32; - else - assert(0); - break; - case U32_TYPE: + if ( pI->is_hi() ) { + d.s64 = (t.s64>>16) + c.s32; + if ( sat_mode ) { + if ( d.s64 > (int)0x7FFFFFFF ) + d.s64 = (int)0x7FFFFFFF; + else if ( d.s64 < (int)0x80000000 ) + d.s64 = (int)0x80000000; + } + } else if ( pI->is_lo() ) d.s64 = t.s32 + c.s32; + else assert(0); + break; + case U32_TYPE: t.u64 = a.u32 * b.u32; - if (pI->is_hi()) - d.u64 = (t.u64 >> 16) + c.u32; - else if (pI->is_lo()) - d.u64 = t.u32 + c.u32; - else - assert(0); + if ( pI->is_hi() ) d.u64 = (t.u64>>16) + c.u32; + else if ( pI->is_lo() ) d.u64 = t.u32 + c.u32; + else assert(0); break; - default: + default: assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void mad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - mad_def(pI, thread, false); +void mad_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + mad_def(pI, thread, false); } -void madp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - mad_def(pI, thread, true); +void madp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + mad_def(pI, thread, true); } -void madc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - mad_def(pI, thread, true); +void madc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + mad_def(pI, thread, true); } -void mad_def(const ptx_instruction *pI, ptx_thread_info *thread, - bool use_carry) { - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t d, t; - - int carry = 0; - int overflow = 0; - - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - - // take the carry bit, it should be the 4th operand - ptx_reg_t carry_bit; - carry_bit.u64 = 0; - if (use_carry) { - const operand_info &carry = pI->operand_lookup(4); - carry_bit = thread->get_operand_value(carry, dst, PRED_TYPE, thread, 0); - carry_bit.pred &= 0x4; - carry_bit.pred >>= 2; - } +void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t d, t; + + int carry=0; + int overflow=0; + + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - unsigned rounding_mode = pI->rounding_mode(); + // take the carry bit, it should be the 4th operand + ptx_reg_t carry_bit; + carry_bit.u64 = 0; + if (use_carry) { + const operand_info &carry = pI->operand_lookup(4); + carry_bit = thread->get_operand_value(carry, dst, PRED_TYPE, thread, 0); + carry_bit.pred &= 0x4; + carry_bit.pred >>=2; + } - switch (i_type) { - case S16_TYPE: + unsigned rounding_mode = pI->rounding_mode(); + + switch ( i_type ) { + case S16_TYPE: t.s32 = a.s16 * b.s16; - if (pI->is_wide()) - d.s32 = t.s32 + c.s32 + carry_bit.pred; - else if (pI->is_hi()) - d.s16 = (t.s32 >> 16) + c.s16 + carry_bit.pred; - else if (pI->is_lo()) - d.s16 = t.s16 + c.s16 + carry_bit.pred; - else - assert(0); - carry = - ((long long int)(t.s32 + c.s32 + carry_bit.pred) & 0x100000000) >> 32; + if ( pI->is_wide() ) d.s32 = t.s32 + c.s32 + carry_bit.pred; + else if ( pI->is_hi() ) d.s16 = (t.s32>>16) + c.s16 + carry_bit.pred; + else if ( pI->is_lo() ) d.s16 = t.s16 + c.s16 + carry_bit.pred; + else assert(0); + carry = ((long long int)(t.s32 + c.s32 + carry_bit.pred)&0x100000000)>>32; break; - case S32_TYPE: + case S32_TYPE: t.s64 = a.s32 * b.s32; - if (pI->is_wide()) - d.s64 = t.s64 + c.s64 + carry_bit.pred; - else if (pI->is_hi()) - d.s32 = (t.s64 >> 32) + c.s32 + carry_bit.pred; - else if (pI->is_lo()) - d.s32 = t.s32 + c.s32 + carry_bit.pred; - else - assert(0); + if ( pI->is_wide() ) d.s64 = t.s64 + c.s64 + carry_bit.pred; + else if ( pI->is_hi() ) d.s32 = (t.s64>>32) + c.s32 + carry_bit.pred; + else if ( pI->is_lo() ) d.s32 = t.s32 + c.s32 + carry_bit.pred; + else assert(0); break; - case S64_TYPE: + case S64_TYPE: t.s64 = a.s64 * b.s64; - assert(!pI->is_wide()); - assert(!pI->is_hi()); - assert(use_carry == false); - if (pI->is_lo()) - d.s64 = t.s64 + c.s64 + carry_bit.pred; - else - assert(0); + assert( !pI->is_wide() ); + assert( !pI->is_hi() ); + assert( use_carry == false); + if ( pI->is_lo() ) d.s64 = t.s64 + c.s64 + carry_bit.pred; + else assert(0); break; - case U16_TYPE: + case U16_TYPE: t.u32 = a.u16 * b.u16; - if (pI->is_wide()) - d.u32 = t.u32 + c.u32 + carry_bit.pred; - else if (pI->is_hi()) - d.u16 = (t.u32 + c.u16 + carry_bit.pred) >> 16; - else if (pI->is_lo()) - d.u16 = t.u16 + c.u16 + carry_bit.pred; - else - assert(0); - carry = ((long long int)((long long int)t.u32 + c.u32 + carry_bit.pred) & - 0x100000000) >> - 32; + if ( pI->is_wide() ) d.u32 = t.u32 + c.u32 + carry_bit.pred; + else if ( pI->is_hi() ) d.u16 = (t.u32 + c.u16 + carry_bit.pred)>>16; + else if ( pI->is_lo() ) d.u16 = t.u16 + c.u16 + carry_bit.pred; + else assert(0); + carry = ((long long int)((long long int)t.u32 + c.u32 + carry_bit.pred)&0x100000000)>>32; break; - case U32_TYPE: + case U32_TYPE: t.u64 = a.u32 * b.u32; - if (pI->is_wide()) - d.u64 = t.u64 + c.u64 + carry_bit.pred; - else if (pI->is_hi()) - d.u32 = (t.u64 + c.u32 + carry_bit.pred) >> 32; - else if (pI->is_lo()) - d.u32 = t.u32 + c.u32 + carry_bit.pred; - else - assert(0); + if ( pI->is_wide() ) d.u64 = t.u64 + c.u64 + carry_bit.pred; + else if ( pI->is_hi() ) d.u32 = (t.u64 + c.u32 + carry_bit.pred)>>32; + else if ( pI->is_lo() ) d.u32 = t.u32 + c.u32 + carry_bit.pred; + else assert(0); break; - case U64_TYPE: + case U64_TYPE: t.u64 = a.u64 * b.u64; - assert(!pI->is_wide()); - assert(!pI->is_hi()); - assert(use_carry == false); - if (pI->is_lo()) - d.u64 = t.u64 + c.u64 + carry_bit.pred; - else - assert(0); - break; - case F16_TYPE: { - // assert(0); - // break; - assert(use_carry == false); - int orig_rm = fegetround(); - switch (rounding_mode) { - case RN_OPTION: - break; - case RZ_OPTION: - fesetround(FE_TOWARDZERO); - break; - default: - assert(0); - break; - } - d.f16 = a.f16 * b.f16 + c.f16; - if (pI->saturation_mode()) { - if (d.f16 < 0) - d.f16 = 0; - else if (d.f16 > 1.0f) - d.f16 = 1.0f; - } - fesetround(orig_rm); - break; - } - case F32_TYPE: { - assert(use_carry == false); - int orig_rm = fegetround(); - switch (rounding_mode) { - case RN_OPTION: - break; - case RZ_OPTION: - fesetround(FE_TOWARDZERO); - break; - default: - assert(0); - break; - } - d.f32 = a.f32 * b.f32 + c.f32; - if (pI->saturation_mode()) { - if (d.f32 < 0) - d.f32 = 0; - else if (d.f32 > 1.0f) - d.f32 = 1.0f; - } - fesetround(orig_rm); - break; - } - case F64_TYPE: - case FF64_TYPE: { - assert(use_carry == false); - int orig_rm = fegetround(); - switch (rounding_mode) { - case RN_OPTION: - break; - case RZ_OPTION: - fesetround(FE_TOWARDZERO); - break; - default: - assert(0); - break; + assert( !pI->is_wide() ); + assert( !pI->is_hi() ); + assert( use_carry == false); + if ( pI->is_lo() ) d.u64 = t.u64 + c.u64 + carry_bit.pred; + else assert(0); + break; + case F16_TYPE:{ + // assert(0); + // break; + assert( use_carry == false); + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + d.f16 = a.f16 * b.f16 + c.f16; + if ( pI->saturation_mode() ) { + if ( d.f16 < 0 ) d.f16 = 0; + else if ( d.f16 > 1.0f ) d.f16 = 1.0f; + } + fesetround( orig_rm ); + break; + } + case F32_TYPE: { + assert( use_carry == false); + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + d.f32 = a.f32 * b.f32 + c.f32; + if ( pI->saturation_mode() ) { + if ( d.f32 < 0 ) d.f32 = 0; + else if ( d.f32 > 1.0f ) d.f32 = 1.0f; + } + fesetround( orig_rm ); + break; + } + case F64_TYPE: case FF64_TYPE: { + assert( use_carry == false); + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + d.f64 = a.f64 * b.f64 + c.f64; + if ( pI->saturation_mode() ) { + if ( d.f64 < 0 ) d.f64 = 0; + else if ( d.f64 > 1.0f ) d.f64 = 1.0; + } + fesetround( orig_rm ); + break; } - d.f64 = a.f64 * b.f64 + c.f64; - if (pI->saturation_mode()) { - if (d.f64 < 0) - d.f64 = 0; - else if (d.f64 > 1.0f) - d.f64 = 1.0; - } - fesetround(orig_rm); - break; - } - default: + default: assert(0); break; - } - thread->set_operand_value(dst, d, i_type, thread, pI, overflow, carry); + } + thread->set_operand_value(dst, d, i_type, thread, pI, overflow, carry); } -bool isNaN(float x) { return std::isnan(x); } +bool isNaN(float x) +{ + return std::isnan(x); +} -bool isNaN(double x) { return std::isnan(x); } +bool isNaN(double x) +{ + return std::isnan(x); +} -void max_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void max_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch (i_type) { - case U16_TYPE: - d.u16 = MY_MAX_I(a.u16, b.u16); - break; - case U32_TYPE: - d.u32 = MY_MAX_I(a.u32, b.u32); - break; - case U64_TYPE: - d.u64 = MY_MAX_I(a.u64, b.u64); - break; - case S16_TYPE: - d.s16 = MY_MAX_I(a.s16, b.s16); - break; - case S32_TYPE: - d.s32 = MY_MAX_I(a.s32, b.s32); - break; - case S64_TYPE: - d.s64 = MY_MAX_I(a.s64, b.s64); - break; - case F32_TYPE: - d.f32 = MY_MAX_F(a.f32, b.f32); - break; - case F64_TYPE: - case FF64_TYPE: - d.f64 = MY_MAX_F(a.f64, b.f64); - break; - default: + + switch ( i_type ) { + case U16_TYPE: d.u16 = MY_MAX_I(a.u16,b.u16); break; + case U32_TYPE: d.u32 = MY_MAX_I(a.u32,b.u32); break; + case U64_TYPE: d.u64 = MY_MAX_I(a.u64,b.u64); break; + case S16_TYPE: d.s16 = MY_MAX_I(a.s16,b.s16); break; + case S32_TYPE: d.s32 = MY_MAX_I(a.s32,b.s32); break; + case S64_TYPE: d.s64 = MY_MAX_I(a.s64,b.s64); break; + case F32_TYPE: d.f32 = MY_MAX_F(a.f32,b.f32); break; + case F64_TYPE: case FF64_TYPE: d.f64 = MY_MAX_F(a.f64,b.f64); break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -void membar_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - // handled by timing simulator +void membar_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + // handled by timing simulator } -void min_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void min_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch (i_type) { - case U16_TYPE: - d.u16 = MY_MIN_I(a.u16, b.u16); - break; - case U32_TYPE: - d.u32 = MY_MIN_I(a.u32, b.u32); - break; - case U64_TYPE: - d.u64 = MY_MIN_I(a.u64, b.u64); - break; - case S16_TYPE: - d.s16 = MY_MIN_I(a.s16, b.s16); - break; - case S32_TYPE: - d.s32 = MY_MIN_I(a.s32, b.s32); - break; - case S64_TYPE: - d.s64 = MY_MIN_I(a.s64, b.s64); - break; - case F32_TYPE: - d.f32 = MY_MIN_F(a.f32, b.f32); - break; - case F64_TYPE: - case FF64_TYPE: - d.f64 = MY_MIN_F(a.f64, b.f64); - break; - default: + + switch ( i_type ) { + case U16_TYPE: d.u16 = MY_MIN_I(a.u16,b.u16); break; + case U32_TYPE: d.u32 = MY_MIN_I(a.u32,b.u32); break; + case U64_TYPE: d.u64 = MY_MIN_I(a.u64,b.u64); break; + case S16_TYPE: d.s16 = MY_MIN_I(a.s16,b.s16); break; + case S32_TYPE: d.s32 = MY_MIN_I(a.s32,b.s32); break; + case S64_TYPE: d.s64 = MY_MIN_I(a.s64,b.s64); break; + case F32_TYPE: d.f32 = MY_MIN_F(a.f32,b.f32); break; + case F64_TYPE: case FF64_TYPE: d.f64 = MY_MIN_F(a.f64,b.f64); break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -void mov_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t data; +void mov_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - assert(src1.is_param_local() == 0); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + assert( src1.is_param_local() == 0 ); - if ((src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && - (i_type != BB128_TYPE) && (i_type != FF64_TYPE)) { - // pack or unpack operation - unsigned nbits_to_move; - ptx_reg_t tmp_bits; + if( (src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && (i_type != BB128_TYPE) && (i_type != FF64_TYPE) ) { + // pack or unpack operation + unsigned nbits_to_move; + ptx_reg_t tmp_bits; - switch (pI->get_type()) { - case B16_TYPE: - nbits_to_move = 16; - break; - case B32_TYPE: - nbits_to_move = 32; - break; - case B64_TYPE: - nbits_to_move = 64; - break; - default: - printf( - "Execution error: mov pack/unpack with unsupported type " - "qualifier\n"); - assert(0); - break; - } + switch( pI->get_type() ) { + case B16_TYPE: nbits_to_move = 16; break; + case B32_TYPE: nbits_to_move = 32; break; + case B64_TYPE: nbits_to_move = 64; break; + default: printf("Execution error: mov pack/unpack with unsupported type qualifier\n"); assert(0); break; + } - if (src1.is_vector()) { - unsigned nelem = src1.get_vect_nelem(); - ptx_reg_t v[4]; - thread->get_vector_operand_values(src1, v, nelem); + if( src1.is_vector() ) { + unsigned nelem = src1.get_vect_nelem(); + ptx_reg_t v[4]; + thread->get_vector_operand_values(src1, v, nelem ); + + unsigned bits_per_src_elem = nbits_to_move / nelem; + for( unsigned i=0; i < nelem; i++ ) { + switch(bits_per_src_elem) { + case 8: tmp_bits.u64 |= ((unsigned long long)(v[i].u8) << (8*i)); break; + case 16: tmp_bits.u64 |= ((unsigned long long)(v[i].u16) << (16*i)); break; + case 32: tmp_bits.u64 |= ((unsigned long long)(v[i].u32) << (32*i)); break; + default: printf("Execution error: mov pack/unpack with unsupported source/dst size ratio (src)\n"); assert(0); break; + } + } + } else { + data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch( pI->get_type() ) { + case B16_TYPE: tmp_bits.u16 = data.u16; break; + case B32_TYPE: tmp_bits.u32 = data.u32; break; + case B64_TYPE: tmp_bits.u64 = data.u64; break; + default: assert(0); break; + } + } - unsigned bits_per_src_elem = nbits_to_move / nelem; - for (unsigned i = 0; i < nelem; i++) { - switch (bits_per_src_elem) { - case 8: - tmp_bits.u64 |= ((unsigned long long)(v[i].u8) << (8 * i)); - break; - case 16: - tmp_bits.u64 |= ((unsigned long long)(v[i].u16) << (16 * i)); - break; - case 32: - tmp_bits.u64 |= ((unsigned long long)(v[i].u32) << (32 * i)); - break; - default: - printf( - "Execution error: mov pack/unpack with unsupported source/dst " - "size ratio (src)\n"); - assert(0); - break; - } + if( dst.is_vector() ) { + unsigned nelem = dst.get_vect_nelem(); + ptx_reg_t v[4]; + unsigned bits_per_dst_elem = nbits_to_move / nelem; + for( unsigned i=0; i < nelem; i++ ) { + switch(bits_per_dst_elem) { + case 8: v[i].u8 = (tmp_bits.u64 >> (8*i)) & ((unsigned long long) 0xFF); break; + case 16: v[i].u16 = (tmp_bits.u64 >> (16*i)) & ((unsigned long long) 0xFFFF); break; + case 32: v[i].u32 = (tmp_bits.u64 >> (32*i)) & ((unsigned long long) 0xFFFFFFFF); break; + default: + printf("Execution error: mov pack/unpack with unsupported source/dst size ratio (dst)\n"); + assert(0); + break; + } + } + thread->set_vector_operand_values(dst,v[0],v[1],v[2],v[3]); + } else { + thread->set_operand_value(dst,tmp_bits, i_type, thread, pI); } - } else { + } else if (i_type == PRED_TYPE and src1.is_literal() == true) { + // in ptx, literal input translate to predicate as 0 = false and 1 = true + // we have adopted the opposite to simplify implementation of zero flags in ptxplus data = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch (pI->get_type()) { - case B16_TYPE: - tmp_bits.u16 = data.u16; - break; - case B32_TYPE: - tmp_bits.u32 = data.u32; - break; - case B64_TYPE: - tmp_bits.u64 = data.u64; - break; - default: - assert(0); - break; - } - } + ptx_reg_t finaldata; + finaldata.pred = (data.u32 == 0)? 1 : 0; // setting zero-flag in predicate + thread->set_operand_value(dst, finaldata, i_type, thread, pI); + } else { - if (dst.is_vector()) { - unsigned nelem = dst.get_vect_nelem(); - ptx_reg_t v[4]; - unsigned bits_per_dst_elem = nbits_to_move / nelem; - for (unsigned i = 0; i < nelem; i++) { - switch (bits_per_dst_elem) { - case 8: - v[i].u8 = (tmp_bits.u64 >> (8 * i)) & ((unsigned long long)0xFF); - break; - case 16: - v[i].u16 = - (tmp_bits.u64 >> (16 * i)) & ((unsigned long long)0xFFFF); - break; - case 32: - v[i].u32 = - (tmp_bits.u64 >> (32 * i)) & ((unsigned long long)0xFFFFFFFF); - break; - default: - printf( - "Execution error: mov pack/unpack with unsupported source/dst " - "size ratio (dst)\n"); - assert(0); - break; - } - } - thread->set_vector_operand_values(dst, v[0], v[1], v[2], v[3]); - } else { - thread->set_operand_value(dst, tmp_bits, i_type, thread, pI); - } - } else if (i_type == PRED_TYPE and src1.is_literal() == true) { - // in ptx, literal input translate to predicate as 0 = false and 1 = true - // we have adopted the opposite to simplify implementation of zero flags in - // ptxplus - data = thread->get_operand_value(src1, dst, i_type, thread, 1); - - ptx_reg_t finaldata; - finaldata.pred = (data.u32 == 0) ? 1 : 0; // setting zero-flag in predicate - thread->set_operand_value(dst, finaldata, i_type, thread, pI); - } else { - data = thread->get_operand_value(src1, dst, i_type, thread, 1); + data = thread->get_operand_value(src1, dst, i_type, thread, 1); - thread->set_operand_value(dst, data, i_type, thread, pI); - } + thread->set_operand_value(dst, data, i_type, thread, pI); + + } } -void mul24_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; +void mul24_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - // src1_data = srcOperandModifiers(src1_data, src1, dst, i_type, thread); - // src2_data = srcOperandModifiers(src2_data, src2, dst, i_type, thread); + //src1_data = srcOperandModifiers(src1_data, src1, dst, i_type, thread); + //src2_data = srcOperandModifiers(src2_data, src2, dst, i_type, thread); - src1_data.mask_and(0, 0x00FFFFFF); - src2_data.mask_and(0, 0x00FFFFFF); + src1_data.mask_and(0,0x00FFFFFF); + src2_data.mask_and(0,0x00FFFFFF); - switch (i_type) { - case S32_TYPE: - if (src1_data.get_bit(23)) src1_data.mask_or(0xFFFFFFFF, 0xFF000000); - if (src2_data.get_bit(23)) src2_data.mask_or(0xFFFFFFFF, 0xFF000000); + switch ( i_type ) { + case S32_TYPE: + if( src1_data.get_bit(23) ) + src1_data.mask_or(0xFFFFFFFF,0xFF000000); + if( src2_data.get_bit(23) ) + src2_data.mask_or(0xFFFFFFFF,0xFF000000); data.s64 = src1_data.s64 * src2_data.s64; break; - case U32_TYPE: + case U32_TYPE: data.u64 = src1_data.u64 * src2_data.u64; break; - default: - printf( - "GPGPU-Sim PTX: Execution error - type mismatch with instruction\n"); + default: + printf("GPGPU-Sim PTX: Execution error - type mismatch with instruction\n"); assert(0); break; - } + } - if (pI->is_hi()) { - data.u64 = data.u64 >> 16; - data.mask_and(0, 0xFFFFFFFF); - } else if (pI->is_lo()) { - data.mask_and(0, 0xFFFFFFFF); - } + if ( pI->is_hi() ) { + data.u64 = data.u64 >> 16; + data.mask_and(0,0xFFFFFFFF); + } else if (pI->is_lo()) { + data.mask_and(0,0xFFFFFFFF); + } - thread->set_operand_value(dst, data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void mul_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t data; +void mul_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - ptx_reg_t d, t; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + ptx_reg_t d, t; - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - unsigned rounding_mode = pI->rounding_mode(); + unsigned rounding_mode = pI->rounding_mode(); - switch (i_type) { - case S16_TYPE: + switch ( i_type ) { + case S16_TYPE: t.s32 = ((int)a.s16) * ((int)b.s16); - if (pI->is_wide()) - d.s32 = t.s32; - else if (pI->is_hi()) - d.s16 = (t.s32 >> 16); - else if (pI->is_lo()) - d.s16 = t.s16; - else - assert(0); + if ( pI->is_wide() ) d.s32 = t.s32; + else if ( pI->is_hi() ) d.s16 = (t.s32>>16); + else if ( pI->is_lo() ) d.s16 = t.s16; + else assert(0); break; - case S32_TYPE: + case S32_TYPE: t.s64 = ((long long)a.s32) * ((long long)b.s32); - if (pI->is_wide()) - d.s64 = t.s64; - else if (pI->is_hi()) - d.s32 = (t.s64 >> 32); - else if (pI->is_lo()) - d.s32 = t.s32; - else - assert(0); + if ( pI->is_wide() ) d.s64 = t.s64; + else if ( pI->is_hi() ) d.s32 = (t.s64>>32); + else if ( pI->is_lo() ) d.s32 = t.s32; + else assert(0); break; - case S64_TYPE: + case S64_TYPE: t.s64 = a.s64 * b.s64; - assert(!pI->is_wide()); - assert(!pI->is_hi()); - if (pI->is_lo()) - d.s64 = t.s64; - else - assert(0); + assert( !pI->is_wide() ); + assert( !pI->is_hi() ); + if ( pI->is_lo() ) d.s64 = t.s64; + else assert(0); break; - case U16_TYPE: + case U16_TYPE: t.u32 = ((unsigned)a.u16) * ((unsigned)b.u16); - if (pI->is_wide()) - d.u32 = t.u32; - else if (pI->is_lo()) - d.u16 = t.u16; - else if (pI->is_hi()) - d.u16 = (t.u32 >> 16); - else - assert(0); + if ( pI->is_wide() ) d.u32 = t.u32; + else if ( pI->is_lo() ) d.u16 = t.u16; + else if ( pI->is_hi() ) d.u16 = (t.u32>>16); + else assert(0); break; - case U32_TYPE: + case U32_TYPE: t.u64 = ((unsigned long long)a.u32) * ((unsigned long long)b.u32); - if (pI->is_wide()) - d.u64 = t.u64; - else if (pI->is_lo()) - d.u32 = t.u32; - else if (pI->is_hi()) - d.u32 = (t.u64 >> 32); - else - assert(0); + if ( pI->is_wide() ) d.u64 = t.u64; + else if ( pI->is_lo() ) d.u32 = t.u32; + else if ( pI->is_hi() ) d.u32 = (t.u64>>32); + else assert(0); break; - case U64_TYPE: + case U64_TYPE: t.u64 = a.u64 * b.u64; - assert(!pI->is_wide()); - assert(!pI->is_hi()); - if (pI->is_lo()) - d.u64 = t.u64; - else - assert(0); - break; - case F16_TYPE: { - // assert(0); - // break; - int orig_rm = fegetround(); - switch (rounding_mode) { - case RN_OPTION: - break; - case RZ_OPTION: - fesetround(FE_TOWARDZERO); - break; - default: - assert(0); - break; - } - - d.f16 = a.f16 * b.f16; - - if (pI->saturation_mode()) { - if (d.f16 < 0) - d.f16 = 0; - else if (d.f16 > 1.0f) - d.f16 = 1.0f; - } - fesetround(orig_rm); - break; - } - case F32_TYPE: { - int orig_rm = fegetround(); - switch (rounding_mode) { - case RN_OPTION: - break; - case RZ_OPTION: - fesetround(FE_TOWARDZERO); - break; - default: - assert(0); - break; - } - - d.f32 = a.f32 * b.f32; - - if (pI->saturation_mode()) { - if (d.f32 < 0) - d.f32 = 0; - else if (d.f32 > 1.0f) - d.f32 = 1.0f; - } - fesetround(orig_rm); - break; - } - case F64_TYPE: - case FF64_TYPE: { - int orig_rm = fegetround(); - switch (rounding_mode) { - case RN_OPTION: - break; - case RZ_OPTION: - fesetround(FE_TOWARDZERO); - break; - default: - assert(0); - break; - } - d.f64 = a.f64 * b.f64; - if (pI->saturation_mode()) { - if (d.f64 < 0) - d.f64 = 0; - else if (d.f64 > 1.0f) - d.f64 = 1.0; + assert( !pI->is_wide() ); + assert( !pI->is_hi() ); + if ( pI->is_lo() ) d.u64 = t.u64; + else assert(0); + break; + case F16_TYPE:{ + //assert(0); + //break; + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + + d.f16 = a.f16 * b.f16; + + if ( pI->saturation_mode() ) { + if ( d.f16 < 0 ) d.f16 = 0; + else if ( d.f16 > 1.0f ) d.f16 = 1.0f; + } + fesetround( orig_rm ); + break; + } + case F32_TYPE: { + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + + d.f32 = a.f32 * b.f32; + + if ( pI->saturation_mode() ) { + if ( d.f32 < 0 ) d.f32 = 0; + else if ( d.f32 > 1.0f ) d.f32 = 1.0f; + } + fesetround( orig_rm ); + break; + } + case F64_TYPE: case FF64_TYPE:{ + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + d.f64 = a.f64 * b.f64; + if ( pI->saturation_mode() ) { + if ( d.f64 < 0 ) d.f64 = 0; + else if ( d.f64 > 1.0f ) d.f64 = 1.0; + } + fesetround( orig_rm ); + break; } - fesetround(orig_rm); - break; - } - default: - assert(0); + default: + assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void neg_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned to_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, to_type, thread, 1); - - switch (to_type) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - data.s64 = 0 - src1_data.s64; - break; // seems buggy, but not (just ignore higher bits) - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - assert(0); - break; - case F16_TYPE: - data.f16 = 0.0f - src1_data.f16; - break; // assert(0); break; - case F32_TYPE: - data.f32 = 0.0f - src1_data.f32; - break; - case F64_TYPE: - case FF64_TYPE: - data.f64 = 0.0f - src1_data.f64; - break; - default: - assert(0); - break; - } - - thread->set_operand_value(dst, data, to_type, thread, pI); -} +void neg_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; -// nandn bitwise negates second operand then bitwise nands with the first -// operand -void nandn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + unsigned to_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, to_type, thread, 1); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - // the way ptxplus handles predicates: 1 = false and 0 = true - if (i_type == PRED_TYPE) - data.pred = (~src1_data.pred & src2_data.pred); - else - data.u64 = ~(src1_data.u64 & ~src2_data.u64); + switch ( to_type ) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + data.s64 = 0 - src1_data.s64; break; // seems buggy, but not (just ignore higher bits) + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + assert(0); break; + case F16_TYPE: data.f16 =0.0f - src1_data.f16; break;//assert(0); break; + case F32_TYPE: data.f32 = 0.0f - src1_data.f32; break; + case F64_TYPE: case FF64_TYPE: data.f64 = 0.0f - src1_data.f64; break; + default: assert(0); break; + } - thread->set_operand_value(dst, data, i_type, thread, pI); + thread->set_operand_value(dst,data, to_type, thread, pI); } -// norn bitwise negates first operand then bitwise ands with the second operand -void norn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +//nandn bitwise negates second operand then bitwise nands with the first operand +void nandn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - // the way ptxplus handles predicates: 1 = false and 0 = true - if (i_type == PRED_TYPE) - data.pred = ~(src1_data.pred & ~(src2_data.pred)); - else - data.u64 = ~(src1_data.u64) & src2_data.u64; + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - thread->set_operand_value(dst, data, i_type, thread, pI); -} - -void not_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = (~src1_data.pred & src2_data.pred); + else + data.u64 = ~(src1_data.u64 & ~src2_data.u64); - switch (i_type) { - case PRED_TYPE: - d.pred = (~(a.pred) & 0x000F); - break; - case B16_TYPE: - d.u16 = ~a.u16; - break; - case B32_TYPE: - d.u32 = ~a.u32; - break; - case B64_TYPE: - d.u64 = ~a.u64; - break; - default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } + thread->set_operand_value(dst,data, i_type, thread, pI); - thread->set_operand_value(dst, d, i_type, thread, pI); } -void or_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +//norn bitwise negates first operand then bitwise ands with the second operand +void norn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - // the way ptxplus handles predicates: 1 = false and 0 = true - if (i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) | ~(src2_data.pred)); - else - data.u64 = src1_data.u64 | src2_data.u64; - - thread->set_operand_value(dst, data, i_type, thread, pI); -} + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); -void orn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = ~(src1_data.pred & ~(src2_data.pred)); + else + data.u64 = ~(src1_data.u64) & src2_data.u64; - // the way ptxplus handles predicates: 1 = false and 0 = true - if (i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) | (src2_data.pred)); - else - data.u64 = src1_data.u64 | ~src2_data.u64; + thread->set_operand_value(dst,data, i_type, thread, pI); - thread->set_operand_value(dst, data, i_type, thread, pI); } -void pmevent_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void popc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src = pI->src1(); +void not_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - src_data = thread->get_operand_value(src, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch (i_type) { - case B32_TYPE: { - std::bitset<32> mask(src_data.u32); - data.u32 = mask.count(); - } break; - case B64_TYPE: { - std::bitset<64> mask(src_data.u64); - data.u32 = mask.count(); - } break; - default: + + switch ( i_type ) { + case PRED_TYPE: d.pred = (~(a.pred) & 0x000F); break; + case B16_TYPE: d.u16 = ~a.u16; break; + case B32_TYPE: d.u32 = ~a.u32; break; + case B64_TYPE: d.u64 = ~a.u64; break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, data, i_type, thread, pI); -} -void prefetch_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void prefetchu_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -int prmt_mode_present(int mode) { - int returnval = 0; - switch (mode) { - case PRMT_F4E_MODE: - case PRMT_B4E_MODE: - case PRMT_RC8_MODE: - case PRMT_RC16_MODE: - case PRMT_ECL_MODE: - case PRMT_ECR_MODE: - returnval = 1; - break; - default: - break; - } - return returnval; -} -int read_byte(int mode, int control, int d_sel_index, signed long long value) { - int returnval = 0; - int prmt_f4e_mode[4][4] = { - {0, 1, 2, 3}, {1, 2, 3, 4}, {2, 3, 4, 5}, {3, 4, 5, 6}}; - int prmt_b4e_mode[4][4] = { - {0, 7, 6, 5}, {1, 0, 7, 6}, {2, 1, 0, 7}, {3, 2, 1, 0}}; - int prmt_rc8_mode[4][4] = { - {0, 0, 0, 0}, {1, 1, 1, 1}, {2, 2, 2, 2}, {3, 3, 3, 3}}; - int prmt_ecl_mode[4][4] = { - {0, 1, 2, 3}, {1, 1, 2, 3}, {2, 2, 2, 3}, {3, 3, 3, 3}}; - int prmt_ecr_mode[4][4] = { - {0, 0, 0, 0}, {0, 1, 1, 1}, {0, 1, 2, 2}, {0, 1, 2, 3}}; - int prmt_rc16_mode[4][4] = { - {0, 1, 0, 1}, {2, 3, 2, 3}, {0, 1, 0, 1}, {2, 3, 2, 3}}; - - if (!prmt_mode_present(mode)) { - if (control & 0x8) { - returnval = 0xff; - } else { - returnval = (value >> (8 * control)) & 0xff; - } - } else { - switch (mode) { - case PRMT_F4E_MODE: - returnval = prmt_f4e_mode[control][d_sel_index]; - break; - case PRMT_B4E_MODE: - returnval = prmt_b4e_mode[control][d_sel_index]; - break; - case PRMT_RC8_MODE: - returnval = prmt_rc8_mode[control][d_sel_index]; - break; - case PRMT_ECL_MODE: - returnval = prmt_ecl_mode[control][d_sel_index]; - break; - case PRMT_ECR_MODE: - returnval = prmt_ecr_mode[control][d_sel_index]; - break; - case PRMT_RC16_MODE: - returnval = prmt_rc16_mode[control][d_sel_index]; - break; - // Change the default from printing "ERROR" to just asserting - default: - assert(false); - } - } - return (returnval << 8 * d_sel_index); +void or_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) | ~(src2_data.pred)); + else + data.u64 = src1_data.u64 | src2_data.u64; + + thread->set_operand_value(dst,data, i_type, thread, pI); } -void prmt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, src3_data, tmpdata, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - - unsigned mode = pI->prmt_op(); - unsigned i_type = pI->get_type(); - - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - - tmpdata.s64 = src1_data.s32 | (src2_data.s64 << 32); - int ctl[4]; - - if (!prmt_mode_present(mode)) { - ctl[0] = (src3_data.s32 >> 0) & 0xf; - ctl[1] = (src3_data.s32 >> 4) & 0xf; - ctl[2] = (src3_data.s32 >> 8) & 0xf; - ctl[3] = (src3_data.s32 >> 12) & 0xf; - } else { - ctl[0] = ctl[1] = ctl[2] = ctl[3] = (src3_data.s32 >> 0) & 0x3; - } +void orn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - data.s32 = 0; - data.s32 = data.s32 | read_byte(mode, ctl[0], 0, tmpdata.s64); // First - // byte-0 - data.s32 = - data.s32 | read_byte(mode, ctl[1], 1, tmpdata.s64); // Second byte-1 - data.s32 = data.s32 | read_byte(mode, ctl[2], 2, tmpdata.s64); // Third - // byte-2 - data.s32 = - data.s32 | read_byte(mode, ctl[3], 3, tmpdata.s64); // Fourth byte-3 - - thread->set_operand_value(dst, data, i_type, thread, pI); + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) | (src2_data.pred)); + else + data.u64 = src1_data.u64 | ~src2_data.u64; + + thread->set_operand_value(dst,data, i_type, thread, pI); } -void rcp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void pmevent_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void popc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src = pI->src1(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src_data = thread->get_operand_value(src, dst, i_type, thread, 1); - switch (i_type) { - case F32_TYPE: + switch ( i_type ) { + case B32_TYPE: { + std::bitset<32> mask(src_data.u32); + data.u32 = mask.count(); + } break; + case B64_TYPE: { + std::bitset<64> mask(src_data.u64); + data.u32 = mask.count(); + } break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst,data, i_type, thread, pI); +} +void prefetch_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void prefetchu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } + +int prmt_mode_present(int mode) +{ + int returnval=0; + switch(mode){ + case PRMT_F4E_MODE: + case PRMT_B4E_MODE: + case PRMT_RC8_MODE: + case PRMT_RC16_MODE: + case PRMT_ECL_MODE: + case PRMT_ECR_MODE: + returnval=1; + break; + default: + break; + } + return returnval; +} +int read_byte(int mode, int control, int d_sel_index, signed long long value){ + + int returnval = 0; + int prmt_f4e_mode[4][4]={{0,1,2,3},{1,2,3,4},{2,3,4,5},{3,4,5,6}}; + int prmt_b4e_mode[4][4]={{0,7,6,5},{1,0,7,6},{2,1,0,7},{3,2,1,0}}; + int prmt_rc8_mode[4][4]={{0,0,0,0},{1,1,1,1},{2,2,2,2},{3,3,3,3}}; + int prmt_ecl_mode[4][4]={{0,1,2,3},{1,1,2,3},{2,2,2,3},{3,3,3,3}}; + int prmt_ecr_mode[4][4]={{0,0,0,0},{0,1,1,1},{0,1,2,2},{0,1,2,3}}; + int prmt_rc16_mode[4][4]={{0,1,0,1},{2,3,2,3},{0,1,0,1},{2,3,2,3}}; + + if(!prmt_mode_present(mode)){ + if(control&0x8){ + returnval=0xff; + } + else{ + returnval= (value>>(8*control)) & 0xff; + } + } + else{ + switch(mode){ + case PRMT_F4E_MODE: returnval=prmt_f4e_mode[control][d_sel_index];break; + case PRMT_B4E_MODE: returnval=prmt_b4e_mode[control][d_sel_index];break; + case PRMT_RC8_MODE: returnval=prmt_rc8_mode[control][d_sel_index];break; + case PRMT_ECL_MODE: returnval=prmt_ecl_mode[control][d_sel_index];break; + case PRMT_ECR_MODE: returnval=prmt_ecr_mode[control][d_sel_index];break; + case PRMT_RC16_MODE: returnval=prmt_rc16_mode[control][d_sel_index];break; + // Change the default from printing "ERROR" to just asserting + default: assert(false); + } + } + return (returnval << 8 * d_sel_index); +} + +void prmt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { + + ptx_reg_t src1_data, src2_data, src3_data,tmpdata,data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + unsigned mode = pI->prmt_op(); + unsigned i_type = pI->get_type(); + + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + + tmpdata.s64=src1_data.s32|(src2_data.s64<<32); + int ctl[4]; + + if(!prmt_mode_present(mode)){ + ctl[0]=(src3_data.s32>>0)&0xf; + ctl[1]=(src3_data.s32>>4)&0xf; + ctl[2]=(src3_data.s32>>8)&0xf; + ctl[3]=(src3_data.s32>>12)&0xf; + } + else{ + ctl[0]=ctl[1]=ctl[2]=ctl[3]=(src3_data.s32>>0)&0x3; + } + + data.s32=0; + data.s32=data.s32|read_byte(mode,ctl[0],0,tmpdata.s64); //First byte-0 + data.s32=data.s32|read_byte(mode,ctl[1],1,tmpdata.s64); //Second byte-1 + data.s32=data.s32|read_byte(mode,ctl[2],2,tmpdata.s64); //Third byte-2 + data.s32=data.s32|read_byte(mode,ctl[3],3,tmpdata.s64); //Fourth byte-3 + + thread->set_operand_value(dst,data, i_type, thread, pI); + + +} + +void rcp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + + switch ( i_type ) { + case F32_TYPE: data.f32 = 1.0f / src1_data.f32; break; - case F64_TYPE: - case FF64_TYPE: + case F64_TYPE: + case FF64_TYPE: data.f64 = 1.0f / src1_data.f64; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, data, i_type, thread, pI); + thread->set_operand_value(dst,data, i_type, thread, pI); } -void red_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} +void red_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void rem_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; +void rem_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch (i_type) { - case S32_TYPE: + switch ( i_type ) { + case S32_TYPE: data.s32 = src1_data.s32 % src2_data.s32; break; - case S64_TYPE: + case S64_TYPE: data.s64 = src1_data.s64 % src2_data.s64; break; - case U32_TYPE: + case U32_TYPE: data.u32 = src1_data.u32 % src2_data.u32; break; - case U64_TYPE: + case U64_TYPE: data.u64 = src1_data.u64 % src2_data.u64; break; - default: - assert(0); - break; - } + default: assert(0); break; + } - thread->set_operand_value(dst, data, i_type, thread, pI); + thread->set_operand_value(dst,data, i_type, thread, pI); } -void ret_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - bool empty = thread->callstack_pop(); - if (empty) { - thread->set_done(); - thread->exitCore(); - thread->registerExit(); - } +void ret_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + bool empty = thread->callstack_pop(); + if( empty ) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); + } } -// Ptxplus version of ret instruction. -void retp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - bool empty = thread->callstack_pop_plus(); - if (empty) { - thread->set_done(); - thread->exitCore(); - thread->registerExit(); - } +//Ptxplus version of ret instruction. +void retp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + bool empty = thread->callstack_pop_plus(); + if( empty ) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); + } } -void rsqrt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - switch (i_type) { - case F32_TYPE: - if (a.f32 < 0) { - d.u64 = 0; - d.u64 = 0x7fc00000; // NaN - } else if (a.f32 == 0) { - d.u64 = 0; - d.u32 = 0x7f800000; // Inf +void rsqrt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + + switch ( i_type ) { + case F32_TYPE: + if ( a.f32 < 0 ) { + d.u64 = 0; + d.u64 = 0x7fc00000; // NaN + } else if ( a.f32 == 0 ) { + d.u64 = 0; + d.u32 = 0x7f800000; // Inf } else - d.f32 = cuda_math::__internal_accurate_fdividef(1.0f, sqrtf(a.f32)); - break; - case F64_TYPE: - case FF64_TYPE: - if (a.f32 < 0) { - d.u64 = 0; - d.u32 = 0x7fc00000; // NaN - float x = d.f32; - d.f64 = (double)x; - } else if (a.f32 == 0) { - d.u64 = 0; - d.u32 = 0x7f800000; // Inf - float x = d.f32; - d.f64 = (double)x; + d.f32 = cuda_math::__internal_accurate_fdividef(1.0f, sqrtf(a.f32)); + break; + case F64_TYPE: + case FF64_TYPE: + if ( a.f32 < 0 ) { + d.u64 = 0; + d.u32 = 0x7fc00000; // NaN + float x = d.f32; + d.f64 = (double)x; + } else if ( a.f32 == 0 ) { + d.u64 = 0; + d.u32 = 0x7f800000; // Inf + float x = d.f32; + d.f64 = (double)x; } else - d.f64 = 1.0 / sqrt(a.f64); + d.f64 = 1.0 / sqrt(a.f64); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -#define SAD(d, a, b, c) d = c + ((a < b) ? (b - a) : (a - b)) +#define SAD(d,a,b,c) d = c + ((adst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); +void sad_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, c, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, i_type, thread, 1); - switch (i_type) { - case U16_TYPE: - SAD(d.u16, a.u16, b.u16, c.u16); - break; - case U32_TYPE: - SAD(d.u32, a.u32, b.u32, c.u32); - break; - case U64_TYPE: - SAD(d.u64, a.u64, b.u64, c.u64); - break; - case S16_TYPE: - SAD(d.s16, a.s16, b.s16, c.s16); - break; - case S32_TYPE: - SAD(d.s32, a.s32, b.s32, c.s32); - break; - case S64_TYPE: - SAD(d.s64, a.s64, b.s64, c.s64); - break; - case F32_TYPE: - SAD(d.f32, a.f32, b.f32, c.f32); - break; - case F64_TYPE: - case FF64_TYPE: - SAD(d.f64, a.f64, b.f64, c.f64); - break; - default: + + switch ( i_type ) { + case U16_TYPE: SAD(d.u16,a.u16,b.u16,c.u16); break; + case U32_TYPE: SAD(d.u32,a.u32,b.u32,c.u32); break; + case U64_TYPE: SAD(d.u64,a.u64,b.u64,c.u64); break; + case S16_TYPE: SAD(d.s16,a.s16,b.s16,c.s16); break; + case S32_TYPE: SAD(d.s32,a.s32,b.s32,c.s32); break; + case S64_TYPE: SAD(d.s64,a.s64,b.s64,c.s64); break; + case F32_TYPE: SAD(d.f32,a.f32,b.f32,c.f32); break; + case F64_TYPE: case FF64_TYPE: SAD(d.f64,a.f64,b.f64,c.f64); break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -void selp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); +void selp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); - ptx_reg_t a, b, c, d; + ptx_reg_t a, b, c, d; - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, i_type, thread, 1); - // predicate value was changed so the lowest bit being set means the zero flag - // is set. - // As a result, the value of c.pred must be inverted to get proper behavior - d = (!(c.pred & 0x0001)) ? a : b; + //predicate value was changed so the lowest bit being set means the zero flag is set. + //As a result, the value of c.pred must be inverted to get proper behavior + d = (!(c.pred & 0x0001))?a:b; - thread->set_operand_value(dst, d, PRED_TYPE, thread, pI); + thread->set_operand_value(dst,d, PRED_TYPE, thread, pI); } -bool isFloat(int type) { - switch (type) { - case F16_TYPE: - case F32_TYPE: - case F64_TYPE: - case FF64_TYPE: +bool isFloat(int type) +{ + switch ( type ) { + case F16_TYPE: + case F32_TYPE: + case F64_TYPE: + case FF64_TYPE: return true; - default: + default: return false; - } + } } -bool CmpOp(int type, ptx_reg_t a, ptx_reg_t b, unsigned cmpop) { - bool t = false; +bool CmpOp( int type, ptx_reg_t a, ptx_reg_t b, unsigned cmpop ) +{ + bool t = false; - switch (type) { - case B16_TYPE: + switch ( type ) { + case B16_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.u16 == b.u16); - break; - case NE_OPTION: - t = (a.u16 != b.u16); - break; - default: - assert(0); + case EQ_OPTION: t = (a.u16 == b.u16); break; + case NE_OPTION: t = (a.u16 != b.u16); break; + default: + assert(0); } - case B32_TYPE: + case B32_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.u32 == b.u32); - break; - case NE_OPTION: - t = (a.u32 != b.u32); - break; - default: - assert(0); + case EQ_OPTION: t = (a.u32 == b.u32); break; + case NE_OPTION: t = (a.u32 != b.u32); break; + default: + assert(0); } - case B64_TYPE: + case B64_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.u64 == b.u64); - break; - case NE_OPTION: - t = (a.u64 != b.u64); - break; - default: - assert(0); + case EQ_OPTION: t = (a.u64 == b.u64); break; + case NE_OPTION: t = (a.u64 != b.u64); break; + default: + assert(0); } break; - case S8_TYPE: - case S16_TYPE: + case S8_TYPE: + case S16_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.s16 == b.s16); - break; - case NE_OPTION: - t = (a.s16 != b.s16); - break; - case LT_OPTION: - t = (a.s16 < b.s16); - break; - case LE_OPTION: - t = (a.s16 <= b.s16); - break; - case GT_OPTION: - t = (a.s16 > b.s16); - break; - case GE_OPTION: - t = (a.s16 >= b.s16); - break; - default: - assert(0); + case EQ_OPTION: t = (a.s16 == b.s16); break; + case NE_OPTION: t = (a.s16 != b.s16); break; + case LT_OPTION: t = (a.s16 < b.s16); break; + case LE_OPTION: t = (a.s16 <= b.s16); break; + case GT_OPTION: t = (a.s16 > b.s16); break; + case GE_OPTION: t = (a.s16 >= b.s16); break; + default: + assert(0); } break; - case S32_TYPE: + case S32_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.s32 == b.s32); - break; - case NE_OPTION: - t = (a.s32 != b.s32); - break; - case LT_OPTION: - t = (a.s32 < b.s32); - break; - case LE_OPTION: - t = (a.s32 <= b.s32); - break; - case GT_OPTION: - t = (a.s32 > b.s32); - break; - case GE_OPTION: - t = (a.s32 >= b.s32); - break; - default: - assert(0); + case EQ_OPTION: t = (a.s32 == b.s32); break; + case NE_OPTION: t = (a.s32 != b.s32); break; + case LT_OPTION: t = (a.s32 < b.s32); break; + case LE_OPTION: t = (a.s32 <= b.s32); break; + case GT_OPTION: t = (a.s32 > b.s32); break; + case GE_OPTION: t = (a.s32 >= b.s32); break; + default: + assert(0); } break; - case S64_TYPE: + case S64_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.s64 == b.s64); - break; - case NE_OPTION: - t = (a.s64 != b.s64); - break; - case LT_OPTION: - t = (a.s64 < b.s64); - break; - case LE_OPTION: - t = (a.s64 <= b.s64); - break; - case GT_OPTION: - t = (a.s64 > b.s64); - break; - case GE_OPTION: - t = (a.s64 >= b.s64); - break; - default: - assert(0); + case EQ_OPTION: t = (a.s64 == b.s64); break; + case NE_OPTION: t = (a.s64 != b.s64); break; + case LT_OPTION: t = (a.s64 < b.s64); break; + case LE_OPTION: t = (a.s64 <= b.s64); break; + case GT_OPTION: t = (a.s64 > b.s64); break; + case GE_OPTION: t = (a.s64 >= b.s64); break; + default: + assert(0); } break; - case U8_TYPE: - case U16_TYPE: + case U8_TYPE: + case U16_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.u16 == b.u16); - break; - case NE_OPTION: - t = (a.u16 != b.u16); - break; - case LT_OPTION: - t = (a.u16 < b.u16); - break; - case LE_OPTION: - t = (a.u16 <= b.u16); - break; - case GT_OPTION: - t = (a.u16 > b.u16); - break; - case GE_OPTION: - t = (a.u16 >= b.u16); - break; - case LO_OPTION: - t = (a.u16 < b.u16); - break; - case LS_OPTION: - t = (a.u16 <= b.u16); - break; - case HI_OPTION: - t = (a.u16 > b.u16); - break; - case HS_OPTION: - t = (a.u16 >= b.u16); - break; - default: - assert(0); + case EQ_OPTION: t = (a.u16 == b.u16); break; + case NE_OPTION: t = (a.u16 != b.u16); break; + case LT_OPTION: t = (a.u16 < b.u16); break; + case LE_OPTION: t = (a.u16 <= b.u16); break; + case GT_OPTION: t = (a.u16 > b.u16); break; + case GE_OPTION: t = (a.u16 >= b.u16); break; + case LO_OPTION: t = (a.u16 < b.u16); break; + case LS_OPTION: t = (a.u16 <= b.u16); break; + case HI_OPTION: t = (a.u16 > b.u16); break; + case HS_OPTION: t = (a.u16 >= b.u16); break; + default: + assert(0); } break; - case U32_TYPE: + case U32_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.u32 == b.u32); - break; - case NE_OPTION: - t = (a.u32 != b.u32); - break; - case LT_OPTION: - t = (a.u32 < b.u32); - break; - case LE_OPTION: - t = (a.u32 <= b.u32); - break; - case GT_OPTION: - t = (a.u32 > b.u32); - break; - case GE_OPTION: - t = (a.u32 >= b.u32); - break; - case LO_OPTION: - t = (a.u32 < b.u32); - break; - case LS_OPTION: - t = (a.u32 <= b.u32); - break; - case HI_OPTION: - t = (a.u32 > b.u32); - break; - case HS_OPTION: - t = (a.u32 >= b.u32); - break; - default: - assert(0); + case EQ_OPTION: t = (a.u32 == b.u32); break; + case NE_OPTION: t = (a.u32 != b.u32); break; + case LT_OPTION: t = (a.u32 < b.u32); break; + case LE_OPTION: t = (a.u32 <= b.u32); break; + case GT_OPTION: t = (a.u32 > b.u32); break; + case GE_OPTION: t = (a.u32 >= b.u32); break; + case LO_OPTION: t = (a.u32 < b.u32); break; + case LS_OPTION: t = (a.u32 <= b.u32); break; + case HI_OPTION: t = (a.u32 > b.u32); break; + case HS_OPTION: t = (a.u32 >= b.u32); break; + default: + assert(0); } break; - case U64_TYPE: + case U64_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.u64 == b.u64); - break; - case NE_OPTION: - t = (a.u64 != b.u64); - break; - case LT_OPTION: - t = (a.u64 < b.u64); - break; - case LE_OPTION: - t = (a.u64 <= b.u64); - break; - case GT_OPTION: - t = (a.u64 > b.u64); - break; - case GE_OPTION: - t = (a.u64 >= b.u64); - break; - case LO_OPTION: - t = (a.u64 < b.u64); - break; - case LS_OPTION: - t = (a.u64 <= b.u64); - break; - case HI_OPTION: - t = (a.u64 > b.u64); - break; - case HS_OPTION: - t = (a.u64 >= b.u64); - break; - default: - assert(0); + case EQ_OPTION: t = (a.u64 == b.u64); break; + case NE_OPTION: t = (a.u64 != b.u64); break; + case LT_OPTION: t = (a.u64 < b.u64); break; + case LE_OPTION: t = (a.u64 <= b.u64); break; + case GT_OPTION: t = (a.u64 > b.u64); break; + case GE_OPTION: t = (a.u64 >= b.u64); break; + case LO_OPTION: t = (a.u64 < b.u64); break; + case LS_OPTION: t = (a.u64 <= b.u64); break; + case HI_OPTION: t = (a.u64 > b.u64); break; + case HS_OPTION: t = (a.u64 >= b.u64); break; + default: + assert(0); } break; - case F16_TYPE: - assert(0); - break; - case F32_TYPE: + case F16_TYPE: assert(0); break; + case F32_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.f32 == b.f32) && !isNaN(a.f32) && !isNaN(b.f32); - break; - case NE_OPTION: - t = (a.f32 != b.f32) && !isNaN(a.f32) && !isNaN(b.f32); - break; - case LT_OPTION: - t = (a.f32 < b.f32) && !isNaN(a.f32) && !isNaN(b.f32); - break; - case LE_OPTION: - t = (a.f32 <= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); - break; - case GT_OPTION: - t = (a.f32 > b.f32) && !isNaN(a.f32) && !isNaN(b.f32); - break; - case GE_OPTION: - t = (a.f32 >= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); - break; - case EQU_OPTION: - t = (a.f32 == b.f32) || isNaN(a.f32) || isNaN(b.f32); - break; - case NEU_OPTION: - t = (a.f32 != b.f32) || isNaN(a.f32) || isNaN(b.f32); - break; - case LTU_OPTION: - t = (a.f32 < b.f32) || isNaN(a.f32) || isNaN(b.f32); - break; - case LEU_OPTION: - t = (a.f32 <= b.f32) || isNaN(a.f32) || isNaN(b.f32); - break; - case GTU_OPTION: - t = (a.f32 > b.f32) || isNaN(a.f32) || isNaN(b.f32); - break; - case GEU_OPTION: - t = (a.f32 >= b.f32) || isNaN(a.f32) || isNaN(b.f32); - break; - case NUM_OPTION: - t = !isNaN(a.f32) && !isNaN(b.f32); - break; - case NAN_OPTION: - t = isNaN(a.f32) || isNaN(b.f32); - break; - default: - assert(0); + case EQ_OPTION: t = (a.f32 == b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; + case NE_OPTION: t = (a.f32 != b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; + case LT_OPTION: t = (a.f32 < b.f32 ) && !isNaN(a.f32) && !isNaN(b.f32); break; + case LE_OPTION: t = (a.f32 <= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; + case GT_OPTION: t = (a.f32 > b.f32 ) && !isNaN(a.f32) && !isNaN(b.f32); break; + case GE_OPTION: t = (a.f32 >= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; + case EQU_OPTION: t = (a.f32 == b.f32) || isNaN(a.f32) || isNaN(b.f32); break; + case NEU_OPTION: t = (a.f32 != b.f32) || isNaN(a.f32) || isNaN(b.f32); break; + case LTU_OPTION: t = (a.f32 < b.f32 ) || isNaN(a.f32) || isNaN(b.f32); break; + case LEU_OPTION: t = (a.f32 <= b.f32) || isNaN(a.f32) || isNaN(b.f32); break; + case GTU_OPTION: t = (a.f32 > b.f32 ) || isNaN(a.f32) || isNaN(b.f32); break; + case GEU_OPTION: t = (a.f32 >= b.f32) || isNaN(a.f32) || isNaN(b.f32); break; + case NUM_OPTION: t = !isNaN(a.f32) && !isNaN(b.f32); break; + case NAN_OPTION: t = isNaN(a.f32) || isNaN(b.f32); break; + default: + assert(0); } break; - case F64_TYPE: - case FF64_TYPE: + case F64_TYPE: + case FF64_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.f64 == b.f64) && !isNaN(a.f64) && !isNaN(b.f64); - break; - case NE_OPTION: - t = (a.f64 != b.f64) && !isNaN(a.f64) && !isNaN(b.f64); - break; - case LT_OPTION: - t = (a.f64 < b.f64) && !isNaN(a.f64) && !isNaN(b.f64); - break; - case LE_OPTION: - t = (a.f64 <= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); - break; - case GT_OPTION: - t = (a.f64 > b.f64) && !isNaN(a.f64) && !isNaN(b.f64); - break; - case GE_OPTION: - t = (a.f64 >= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); - break; - case EQU_OPTION: - t = (a.f64 == b.f64) || isNaN(a.f64) || isNaN(b.f64); - break; - case NEU_OPTION: - t = (a.f64 != b.f64) || isNaN(a.f64) || isNaN(b.f64); - break; - case LTU_OPTION: - t = (a.f64 < b.f64) || isNaN(a.f64) || isNaN(b.f64); - break; - case LEU_OPTION: - t = (a.f64 <= b.f64) || isNaN(a.f64) || isNaN(b.f64); - break; - case GTU_OPTION: - t = (a.f64 > b.f64) || isNaN(a.f64) || isNaN(b.f64); - break; - case GEU_OPTION: - t = (a.f64 >= b.f64) || isNaN(a.f64) || isNaN(b.f64); - break; - case NUM_OPTION: - t = !isNaN(a.f64) && !isNaN(b.f64); - break; - case NAN_OPTION: - t = isNaN(a.f64) || isNaN(b.f64); - break; - default: - assert(0); + case EQ_OPTION: t = (a.f64 == b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; + case NE_OPTION: t = (a.f64 != b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; + case LT_OPTION: t = (a.f64 < b.f64 ) && !isNaN(a.f64) && !isNaN(b.f64); break; + case LE_OPTION: t = (a.f64 <= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; + case GT_OPTION: t = (a.f64 > b.f64 ) && !isNaN(a.f64) && !isNaN(b.f64); break; + case GE_OPTION: t = (a.f64 >= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; + case EQU_OPTION: t = (a.f64 == b.f64) || isNaN(a.f64) || isNaN(b.f64); break; + case NEU_OPTION: t = (a.f64 != b.f64) || isNaN(a.f64) || isNaN(b.f64); break; + case LTU_OPTION: t = (a.f64 < b.f64 ) || isNaN(a.f64) || isNaN(b.f64); break; + case LEU_OPTION: t = (a.f64 <= b.f64) || isNaN(a.f64) || isNaN(b.f64); break; + case GTU_OPTION: t = (a.f64 > b.f64 ) || isNaN(a.f64) || isNaN(b.f64); break; + case GEU_OPTION: t = (a.f64 >= b.f64) || isNaN(a.f64) || isNaN(b.f64); break; + case NUM_OPTION: t = !isNaN(a.f64) && !isNaN(b.f64); break; + case NAN_OPTION: t = isNaN(a.f64) || isNaN(b.f64); break; + default: + assert(0); } break; - default: - assert(0); - break; - } + default: assert(0); break; + } - return t; + return t; } -void setp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, b; +void setp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b; - int t = 0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + int t=0; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - assert(pI->get_num_operands() < - 4); // or need to deal with "c" operand / boolOp + assert( pI->get_num_operands() < 4 ); // or need to deal with "c" operand / boolOp - unsigned type = pI->get_type(); - unsigned cmpop = pI->get_cmpop(); - a = thread->get_operand_value(src1, dst, type, thread, 1); - b = thread->get_operand_value(src2, dst, type, thread, 1); + unsigned type = pI->get_type(); + unsigned cmpop = pI->get_cmpop(); + a = thread->get_operand_value(src1, dst, type, thread, 1); + b = thread->get_operand_value(src2, dst, type, thread, 1); - t = CmpOp(type, a, b, cmpop); + t = CmpOp(type,a,b,cmpop); - ptx_reg_t data; + ptx_reg_t data; - // the way ptxplus handles the zero flag, 1 = false and 0 = true - data.pred = - (t == - 0); // inverting predicate since ptxplus uses "1" for a set zero flag + //the way ptxplus handles the zero flag, 1 = false and 0 = true + data.pred = (t==0); //inverting predicate since ptxplus uses "1" for a set zero flag - thread->set_operand_value(dst, data, PRED_TYPE, thread, pI); + thread->set_operand_value(dst,data, PRED_TYPE, thread, pI); } -void set_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, b; +void set_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b; - int t = 0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + int t=0; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - assert(pI->get_num_operands() < - 4); // or need to deal with "c" operand / boolOp + assert( pI->get_num_operands() < 4 ); // or need to deal with "c" operand / boolOp - unsigned src_type = pI->get_type2(); - unsigned cmpop = pI->get_cmpop(); + unsigned src_type = pI->get_type2(); + unsigned cmpop = pI->get_cmpop(); - a = thread->get_operand_value(src1, dst, src_type, thread, 1); - b = thread->get_operand_value(src2, dst, src_type, thread, 1); + a = thread->get_operand_value(src1, dst, src_type, thread, 1); + b = thread->get_operand_value(src2, dst, src_type, thread, 1); - // Take abs of first operand if needed - if (pI->is_abs()) { - switch (src_type) { - case S16_TYPE: - a.s16 = my_abs(a.s16); - break; - case S32_TYPE: - a.s32 = my_abs(a.s32); - break; - case S64_TYPE: - a.s64 = my_abs(a.s64); - break; - case U16_TYPE: - a.u16 = a.u16; - break; - case U32_TYPE: - a.u32 = my_abs(a.u32); - break; - case U64_TYPE: - a.u64 = my_abs(a.u64); - break; - case F32_TYPE: - a.f32 = my_abs(a.f32); - break; - case F64_TYPE: - case FF64_TYPE: - a.f64 = my_abs(a.f64); - break; + // Take abs of first operand if needed + if(pI->is_abs()) { + switch ( src_type ) { + case S16_TYPE: a.s16 = my_abs(a.s16); break; + case S32_TYPE: a.s32 = my_abs(a.s32); break; + case S64_TYPE: a.s64 = my_abs(a.s64); break; + case U16_TYPE: a.u16 = a.u16; break; + case U32_TYPE: a.u32 = my_abs(a.u32); break; + case U64_TYPE: a.u64 = my_abs(a.u64); break; + case F32_TYPE: a.f32 = my_abs(a.f32); break; + case F64_TYPE: case FF64_TYPE: a.f64 = my_abs(a.f64); break; default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - } - - t = CmpOp(src_type, a, b, cmpop); - - ptx_reg_t data; - if (isFloat(pI->get_type())) { - data.f32 = (t != 0) ? 1.0f : 0.0f; - } else { - data.u32 = (t != 0) ? 0xFFFFFFFF : 0; - } - - thread->set_operand_value(dst, data, pI->get_type(), thread, pI); -} - -void shfl_impl(const ptx_instruction *pI, core_t *core, warp_inst_t inst) { - unsigned i_type = pI->get_type(); - int tid; - - if (core->get_gpu()->is_functional_sim()) - tid = inst.warp_id_func() * core->get_warp_size(); - else - tid = inst.warp_id() * core->get_warp_size(); - - ptx_thread_info *thread = core->get_thread_info()[tid]; - ptx_warp_info *warp_info = thread->m_warp_info; - int lane = warp_info->get_done_threads(); - thread = core->get_thread_info()[tid + lane]; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - int bval = (thread->get_operand_value(src2, dst, i_type, thread, 1)).u32; - int cval = (thread->get_operand_value(src3, dst, i_type, thread, 1)).u32; - int mask = cval >> 8; - bval &= 0x1F; - cval &= 0x1F; - - int maxLane = (lane & mask) | (cval & ~mask); - int minLane = lane & mask; - - int src_idx; - unsigned p; - switch (pI->shfl_op()) { - case UP_OPTION: - src_idx = lane - bval; - p = (src_idx >= maxLane); - break; - case DOWN_OPTION: - src_idx = lane + bval; - p = (src_idx <= maxLane); - break; - case BFLY_OPTION: - src_idx = lane ^ bval; - p = (src_idx <= maxLane); - break; - case IDX_OPTION: - src_idx = minLane | (bval & ~mask); - p = (src_idx <= maxLane); - break; - default: - printf("GPGPU-Sim PTX: ERROR: Invalid shfl option\n"); - assert(0); - break; - } - // copy from own lane - if (!p) src_idx = lane; - - // copy input from lane src_idx - ptx_reg_t data; - if (inst.active(src_idx)) { - ptx_thread_info *source = core->get_thread_info()[tid + src_idx]; - data = source->get_operand_value(src1, dst, i_type, source, 1); - } else { - printf( - "GPGPU-Sim PTX: WARNING: shfl input value unpredictable for inactive " - "threads in a warp\n"); - data.u32 = 0; - } - thread->set_operand_value(dst, data, i_type, thread, pI); - - /* - TODO: deal with predicates appropriately using the following pseudocode: - if (!isGuardPredicateTrue(src_idx)) { - printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for - predicated-off threads in a warp\n"); - } - if (dest predicate selected) data.pred = p; - */ - - // keep track of the number of threads that have executed in the warp - warp_info->inc_done_threads(); - if (warp_info->get_done_threads() == inst.active_count()) { - warp_info->reset_done_threads(); - } -} - -void shl_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - - switch (i_type) { - case B16_TYPE: - case U16_TYPE: - if (b.u16 >= 16) - d.u16 = 0; + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + } + + t = CmpOp(src_type,a,b,cmpop); + + ptx_reg_t data; + if ( isFloat(pI->get_type()) ) { + data.f32 = (t!=0)?1.0f:0.0f; + } else { + data.u32 = (t!=0)?0xFFFFFFFF:0; + } + + thread->set_operand_value(dst, data, pI->get_type(), thread, pI); + +} + +void shfl_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) +{ + unsigned i_type = pI->get_type(); + int tid; + + if(core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); + + ptx_thread_info *thread = core->get_thread_info()[tid]; + ptx_warp_info *warp_info = thread->m_warp_info; + int lane = warp_info->get_done_threads(); + thread = core->get_thread_info()[tid + lane]; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + int bval = (thread->get_operand_value(src2, dst, i_type, thread, 1)).u32; + int cval = (thread->get_operand_value(src3, dst, i_type, thread, 1)).u32; + int mask = cval >> 8; + bval &= 0x1F; + cval &= 0x1F; + + int maxLane = (lane & mask) | (cval & ~mask); + int minLane = lane & mask; + + int src_idx; + unsigned p; + switch(pI->shfl_op()) { + case UP_OPTION: + src_idx = lane - bval; + p = (src_idx >= maxLane); + break; + case DOWN_OPTION: + src_idx = lane + bval; + p = (src_idx <= maxLane); + break; + case BFLY_OPTION: + src_idx = lane ^ bval; + p = (src_idx <= maxLane); + break; + case IDX_OPTION: + src_idx = minLane | (bval & ~mask); + p = (src_idx <= maxLane); + break; + default: + printf("GPGPU-Sim PTX: ERROR: Invalid shfl option\n"); + assert(0); + break; + } + // copy from own lane + if (!p) src_idx = lane; + + // copy input from lane src_idx + ptx_reg_t data; + if (inst.active(src_idx)) { + ptx_thread_info *source = core->get_thread_info()[tid + src_idx]; + data = source->get_operand_value(src1, dst, i_type, source, 1); + } else { + printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for inactive threads in a warp\n"); + data.u32 = 0; + } + thread->set_operand_value(dst, data, i_type, thread, pI); + + /* + TODO: deal with predicates appropriately using the following pseudocode: + if (!isGuardPredicateTrue(src_idx)) { + printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for predicated-off threads in a warp\n"); + } + if (dest predicate selected) data.pred = p; + */ + + // keep track of the number of threads that have executed in the warp + warp_info->inc_done_threads(); + if (warp_info->get_done_threads() == inst.active_count()) { + warp_info->reset_done_threads(); + } +} + +void shl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + + switch ( i_type ) { + case B16_TYPE: + case U16_TYPE: + if ( b.u16 >= 16 ) + d.u16 = 0; else - d.u16 = (unsigned short)((a.u16 << b.u16) & 0xFFFF); + d.u16 = (unsigned short) ((a.u16 << b.u16) & 0xFFFF); break; - case B32_TYPE: - case U32_TYPE: - if (b.u32 >= 32) - d.u32 = 0; + case B32_TYPE: + case U32_TYPE: + if ( b.u32 >= 32 ) + d.u32 = 0; else - d.u32 = (unsigned)((a.u32 << b.u32) & 0xFFFFFFFF); + d.u32 = (unsigned) ((a.u32 << b.u32) & 0xFFFFFFFF); break; - case B64_TYPE: - case U64_TYPE: - if (b.u32 >= 64) - d.u64 = 0; + case B64_TYPE: + case U64_TYPE: + if ( b.u32 >= 64 ) + d.u64 = 0; else - d.u64 = (a.u64 << b.u64); + d.u64 = (a.u64 << b.u64); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void shr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - - switch (i_type) { - case U16_TYPE: - case B16_TYPE: - if (b.u16 < 16) - d.u16 = (unsigned short)((a.u16 >> b.u16) & 0xFFFF); +void shr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + + + switch ( i_type ) { + case U16_TYPE: + case B16_TYPE: + if ( b.u16 < 16 ) + d.u16 = (unsigned short) ((a.u16 >> b.u16) & 0xFFFF); else - d.u16 = 0; + d.u16 = 0; break; - case U32_TYPE: - case B32_TYPE: - if (b.u32 < 32) - d.u32 = (unsigned)((a.u32 >> b.u32) & 0xFFFFFFFF); + case U32_TYPE: + case B32_TYPE: + if ( b.u32 < 32 ) + d.u32 = (unsigned) ((a.u32 >> b.u32) & 0xFFFFFFFF); else - d.u32 = 0; + d.u32 = 0; break; - case U64_TYPE: - case B64_TYPE: - if (b.u32 < 64) - d.u64 = (a.u64 >> b.u64); + case U64_TYPE: + case B64_TYPE: + if ( b.u32 < 64 ) + d.u64 = (a.u64 >> b.u64); else - d.u64 = 0; + d.u64 = 0; break; - case S16_TYPE: - if (b.u16 < 16) - d.s64 = (a.s16 >> b.s16); + case S16_TYPE: + if ( b.u16 < 16 ) + d.s64 = (a.s16 >> b.s16); else { - if (a.s16 < 0) { - d.s64 = -1; - } else { - d.s64 = 0; - } + if ( a.s16 < 0 ) { + d.s64 = -1; + } else { + d.s64 = 0; + } } break; - case S32_TYPE: - if (b.u32 < 32) - d.s64 = (a.s32 >> b.s32); + case S32_TYPE: + if ( b.u32 < 32 ) + d.s64 = (a.s32 >> b.s32); else { - if (a.s32 < 0) { - d.s64 = -1; - } else { - d.s64 = 0; - } + if ( a.s32 < 0 ) { + d.s64 = -1; + } else { + d.s64 = 0; + } } break; - case S64_TYPE: - if (b.u64 < 64) - d.s64 = (a.s64 >> b.u64); + case S64_TYPE: + if ( b.u64 < 64 ) + d.s64 = (a.s64 >> b.u64); else { - if (a.s64 < 0) { - if (b.s32 < 0) { - d.u64 = -1; - d.s32 = 0; - } else { - d.s64 = -1; - } - } else { - d.s64 = 0; - } + if ( a.s64 < 0 ) { + if ( b.s32 < 0 ) { + d.u64 = -1; + d.s32 = 0; + } else { + d.s64 = -1; + } + } else { + d.s64 = 0; + } } break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -void sin_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void sin_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch (i_type) { - case F32_TYPE: + + switch ( i_type ) { + case F32_TYPE: d.f32 = sin(a.f32); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - - thread->set_operand_value(dst, d, i_type, thread, pI); -} - -void slct_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - - ptx_reg_t a, b, c, d; - - unsigned i_type = pI->get_type(); - unsigned c_type = pI->get_type2(); - bool t = false; - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, c_type, thread, 1); - - switch (c_type) { - case S32_TYPE: - t = c.s32 >= 0; - break; - case F32_TYPE: - t = c.f32 >= 0; - break; - default: - assert(0); - } - - switch (i_type) { - case B16_TYPE: - case S16_TYPE: - case U16_TYPE: - d.u16 = t ? a.u16 : b.u16; - break; - case F32_TYPE: - case B32_TYPE: - case S32_TYPE: - case U32_TYPE: - d.u32 = t ? a.u32 : b.u32; - break; - case F64_TYPE: - case FF64_TYPE: - case B64_TYPE: - case S64_TYPE: - case U64_TYPE: - d.u64 = t ? a.u64 : b.u64; - break; - default: - assert(0); - } - - thread->set_operand_value(dst, d, i_type, thread, pI); -} - -void sqrt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - switch (i_type) { - case F32_TYPE: - if (a.f32 < 0) - d.f32 = nanf(""); + assert(0); + break; + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +void slct_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + ptx_reg_t a, b, c, d; + + unsigned i_type = pI->get_type(); + unsigned c_type = pI->get_type2(); + bool t = false; + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, c_type, thread, 1); + + switch ( c_type ) { + case S32_TYPE: t = c.s32 >= 0; break; + case F32_TYPE: t = c.f32 >= 0; break; + default: assert(0); + } + + switch ( i_type ) { + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: d.u16 = t?a.u16:b.u16; break; + case F32_TYPE: + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: d.u32 = t?a.u32:b.u32; break; + case F64_TYPE: + case FF64_TYPE: + case B64_TYPE: + case S64_TYPE: + case U64_TYPE: d.u64 = t?a.u64:b.u64; break; + default: assert(0); + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +void sqrt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + + switch ( i_type ) { + case F32_TYPE: + if ( a.f32 < 0 ) + d.f32 = nanf(""); else - d.f32 = sqrt(a.f32); - break; - case F64_TYPE: - case FF64_TYPE: - if (a.f64 < 0) - d.f64 = nan(""); + d.f32 = sqrt(a.f32); break; + case F64_TYPE: + case FF64_TYPE: + if ( a.f64 < 0 ) + d.f64 = nan(""); else - d.f64 = sqrt(a.f64); - break; - default: + d.f64 = sqrt(a.f64); break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } - - thread->set_operand_value(dst, d, i_type, thread, pI); -} - -void sst_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_instruction *cpI = const_cast(pI); // constant - const operand_info &dst = cpI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - unsigned type = pI->get_type(); - ptx_reg_t dst_data = thread->get_operand_value(dst, dst, type, thread, 1); - ptx_reg_t src1_data = thread->get_operand_value(src1, src1, type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, src1, type, thread, 1); - ptx_reg_t src3_data = thread->get_operand_value(src3, src1, type, thread, 1); - memory_space_t space = pI->get_space(); - memory_space *mem = NULL; - addr_t addr = - src2_data.u32 * 4; // this assumes sstarr memory starts at address 0 - ptx_cta_info *cta_info = thread->m_cta_info; - - decode_space(space, thread, src1, mem, addr); - - size_t size; - int t; - type_info_key::type_decode(type, size, t); - - // store data in sstarr memory - mem->write(addr, size / 8, &src3_data.s64, thread, pI); - - // sync threads - cpI->set_bar_id(16); // use 16 for sst because bar uses an int from 0-15 - - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; - thread->m_last_dram_callback.function = bar_callback; - thread->m_last_dram_callback.instruction = cpI; - - // the last thread that executes loads all of the data back from sstarr memory - int NUM_THREADS = cta_info->num_threads(); - cta_info->inc_bar_threads(); - if (NUM_THREADS == cta_info->get_bar_threads()) { - unsigned offset = 0; - addr = 0; - ptx_reg_t data; - float sstarr_fdata[NUM_THREADS]; - signed long long sstarr_ldata[NUM_THREADS]; - // loop through all of the threads - for (int tid = 0; tid < NUM_THREADS; tid++) { - data.u64 = 0; - mem->read(addr + (tid * 4), size / 8, &data.s64); - sstarr_fdata[tid] = data.f32; - sstarr_ldata[tid] = data.s64; - } - - // squeeze the zeros out of the array and store data back into original - // array - mem = NULL; - addr = src1_data.u32; - space.set_type(global_space); - decode_space(space, thread, src1, mem, addr); - // store nonzero entries and indices - for (int tid = 0; tid < NUM_THREADS; tid++) { - if (sstarr_fdata[tid] != 0) { - float ftid = (float)tid; - mem->write(addr + (offset * 4), size / 8, &sstarr_ldata[tid], thread, - pI); - mem->write(addr + ((NUM_THREADS + offset) * 4), size / 8, &ftid, thread, - pI); - offset++; + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +void sst_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_instruction * cpI = const_cast(pI); // constant + const operand_info &dst = cpI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + unsigned type = pI->get_type(); + ptx_reg_t dst_data = thread->get_operand_value(dst, dst, type, thread, 1); + ptx_reg_t src1_data = thread->get_operand_value(src1, src1, type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, src1, type, thread, 1); + ptx_reg_t src3_data = thread->get_operand_value(src3, src1, type, thread, 1); + memory_space_t space = pI->get_space(); + memory_space *mem = NULL; + addr_t addr = src2_data.u32 * 4; // this assumes sstarr memory starts at address 0 + ptx_cta_info *cta_info = thread->m_cta_info; + + decode_space(space,thread,src1,mem,addr); + + size_t size; + int t; + type_info_key::type_decode(type,size,t); + + // store data in sstarr memory + mem->write(addr,size/8,&src3_data.s64,thread,pI); + + // sync threads + cpI->set_bar_id(16); // use 16 for sst because bar uses an int from 0-15 + + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; + thread->m_last_dram_callback.function = bar_callback; + thread->m_last_dram_callback.instruction = cpI; + + // the last thread that executes loads all of the data back from sstarr memory + int NUM_THREADS = cta_info->num_threads(); + cta_info->inc_bar_threads(); + if (NUM_THREADS == cta_info->get_bar_threads()) { + unsigned offset = 0; + addr = 0; + ptx_reg_t data; + float sstarr_fdata[NUM_THREADS]; + signed long long sstarr_ldata[NUM_THREADS]; + // loop through all of the threads + for (int tid = 0; tid < NUM_THREADS; tid++) { + data.u64=0; + mem->read(addr+(tid*4),size/8,&data.s64); + sstarr_fdata[tid] = data.f32; + sstarr_ldata[tid] = data.s64; + } + + // squeeze the zeros out of the array and store data back into original array + mem = NULL; + addr = src1_data.u32; + space.set_type(global_space); + decode_space(space,thread,src1,mem,addr); + // store nonzero entries and indices + for (int tid = 0; tid < NUM_THREADS; tid++) { + if (sstarr_fdata[tid] != 0) { + float ftid = (float)tid; + mem->write(addr+(offset*4),size/8,&sstarr_ldata[tid],thread,pI); + mem->write(addr+((NUM_THREADS+offset)*4),size/8,&ftid,thread,pI); + offset++; + } + } + // store the number of nonzero elements in the array + data = thread->get_operand_value(src1, dst, type, thread, 1); + data.s64 += 4*(offset-1); + thread->set_operand_value(dst, data, type, thread, pI); + + // fill the rest of the array with zeros (dst should always have a 0 in it) + while (offset < NUM_THREADS) { + mem->write(addr+(offset*4),size/8,&dst_data.s64,thread,pI); + offset++; + } + + cta_info->reset_bar_threads(); + thread->m_last_effective_address = addr+(NUM_THREADS-1)*4; + thread->m_last_memory_space = space; + } +} + +void ssy_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + //printf("Execution Warning: unimplemented ssy instruction is treated as a nop\n"); + // TODO: add implementation +} + +void st_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); //may be scalar or vector of regs + unsigned type = pI->get_type(); + ptx_reg_t addr_reg = thread->get_operand_value(dst, dst, type, thread, 1); + ptx_reg_t data; + memory_space_t space = pI->get_space(); + unsigned vector_spec = pI->get_vector(); + + memory_space *mem = NULL; + addr_t addr = addr_reg.u32; + + decode_space(space,thread,dst,mem,addr); + + size_t size; + int t; + type_info_key::type_decode(type,size,t); + + if (!vector_spec) { + data = thread->get_operand_value(src1, dst, type, thread, 1); + mem->write(addr,size/8,&data.s64,thread,pI); + } else { + if (vector_spec == V2_TYPE) { + ptx_reg_t* ptx_regs = new ptx_reg_t[2]; + thread->get_vector_operand_values(src1, ptx_regs, 2); + mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); + mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); + delete [] ptx_regs; } - } - // store the number of nonzero elements in the array - data = thread->get_operand_value(src1, dst, type, thread, 1); - data.s64 += 4 * (offset - 1); - thread->set_operand_value(dst, data, type, thread, pI); - - // fill the rest of the array with zeros (dst should always have a 0 in it) - while (offset < NUM_THREADS) { - mem->write(addr + (offset * 4), size / 8, &dst_data.s64, thread, pI); - offset++; - } - - cta_info->reset_bar_threads(); - thread->m_last_effective_address = addr + (NUM_THREADS - 1) * 4; - thread->m_last_memory_space = space; - } -} - -void ssy_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - // printf("Execution Warning: unimplemented ssy instruction is treated as a - // nop\n"); - // TODO: add implementation -} - -void st_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); // may be scalar or vector of regs - unsigned type = pI->get_type(); - ptx_reg_t addr_reg = thread->get_operand_value(dst, dst, type, thread, 1); - ptx_reg_t data; - memory_space_t space = pI->get_space(); - unsigned vector_spec = pI->get_vector(); - - memory_space *mem = NULL; - addr_t addr = addr_reg.u32; - - decode_space(space, thread, dst, mem, addr); - - size_t size; - int t; - type_info_key::type_decode(type, size, t); - - if (!vector_spec) { - data = thread->get_operand_value(src1, dst, type, thread, 1); - mem->write(addr, size / 8, &data.s64, thread, pI); - } else { - if (vector_spec == V2_TYPE) { - ptx_reg_t *ptx_regs = new ptx_reg_t[2]; - thread->get_vector_operand_values(src1, ptx_regs, 2); - mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); - mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); - delete[] ptx_regs; - } - if (vector_spec == V3_TYPE) { - ptx_reg_t *ptx_regs = new ptx_reg_t[3]; - thread->get_vector_operand_values(src1, ptx_regs, 3); - mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); - mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); - mem->write(addr + 2 * size / 8, size / 8, &ptx_regs[2].s64, thread, pI); - delete[] ptx_regs; - } - if (vector_spec == V4_TYPE) { - ptx_reg_t *ptx_regs = new ptx_reg_t[4]; - thread->get_vector_operand_values(src1, ptx_regs, 4); - mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); - mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); - mem->write(addr + 2 * size / 8, size / 8, &ptx_regs[2].s64, thread, pI); - mem->write(addr + 3 * size / 8, size / 8, &ptx_regs[3].s64, thread, pI); - delete[] ptx_regs; - } - } - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; + if (vector_spec == V3_TYPE) { + ptx_reg_t* ptx_regs = new ptx_reg_t[3]; + thread->get_vector_operand_values(src1, ptx_regs, 3); + mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); + mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); + mem->write(addr+2*size/8,size/8,&ptx_regs[2].s64,thread,pI); + delete [] ptx_regs; + } + if (vector_spec == V4_TYPE) { + ptx_reg_t* ptx_regs = new ptx_reg_t[4]; + thread->get_vector_operand_values(src1, ptx_regs, 4); + mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); + mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); + mem->write(addr+2*size/8,size/8,&ptx_regs[2].s64,thread,pI); + mem->write(addr+3*size/8,size/8,&ptx_regs[3].s64,thread,pI); + delete [] ptx_regs; + } + } + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; } -void sub_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t data; - int overflow = 0; - int carry = 0; +void sub_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t data; + int overflow = 0; + int carry = 0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - // performs addition. Sets carry and overflow if needed. - // the constant is added in during subtraction so the carry bit is set - // properly. - switch (i_type) { - case S8_TYPE: + //performs addition. Sets carry and overflow if needed. + //the constant is added in during subtraction so the carry bit is set properly. + switch ( i_type ) { + case S8_TYPE: data.s64 = (src1_data.s64 & 0xFF) - (src2_data.s64 & 0xFF) + 0x100; - if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) != 0) { - overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; - } - carry = (data.s32 & 0x100) >> 8; + if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) != 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } + carry = (data.s32 & 0x100)>>8; break; - case S16_TYPE: + case S16_TYPE: data.s64 = (src1_data.s64 & 0xFFFF) - (src2_data.s64 & 0xFFFF) + 0x10000; - if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) != 0) { - overflow = - ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; - } - carry = (data.s32 & 0x10000) >> 16; - break; - case S32_TYPE: - data.s64 = (src1_data.s64 & 0xFFFFFFFF) - (src2_data.s64 & 0xFFFFFFFF) + - 0x100000000; - if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) != 0) { - overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 - ? 0 - : 1; - } - carry = ((data.u64) >> 32) & 0x0001; - break; - case S64_TYPE: - data.s64 = src1_data.s64 - src2_data.s64; - break; - case B8_TYPE: - case U8_TYPE: + if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) != 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } + carry = (data.s32 & 0x10000)>>16; + break; + case S32_TYPE: + data.s64 = (src1_data.s64 & 0xFFFFFFFF) - (src2_data.s64 & 0xFFFFFFFF) + 0x100000000; + if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) != 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } + carry = ((data.u64)>>32) & 0x0001; + break; + case S64_TYPE: + data.s64 = src1_data.s64 - src2_data.s64; break; + case B8_TYPE: + case U8_TYPE: data.u64 = (src1_data.u64 & 0xFF) - (src2_data.u64 & 0xFF) + 0x100; - carry = (data.u64 & 0x100) >> 8; + carry = (data.u64 & 0x100)>>8; break; - case B16_TYPE: - case U16_TYPE: + case B16_TYPE: + case U16_TYPE: data.u64 = (src1_data.u64 & 0xFFFF) - (src2_data.u64 & 0xFFFF) + 0x10000; - carry = (data.u64 & 0x10000) >> 16; - break; - case B32_TYPE: - case U32_TYPE: - data.u64 = (src1_data.u64 & 0xFFFFFFFF) - (src2_data.u64 & 0xFFFFFFFF) + - 0x100000000; - carry = (data.u64 & 0x100000000) >> 32; + carry = (data.u64 & 0x10000)>>16; break; - case B64_TYPE: - case U64_TYPE: - data.u64 = src1_data.u64 - src2_data.u64; + case B32_TYPE: + case U32_TYPE: + data.u64 = (src1_data.u64 & 0xFFFFFFFF) - (src2_data.u64 & 0xFFFFFFFF) + 0x100000000; + carry = (data.u64 & 0x100000000)>>32; break; - case F16_TYPE: - data.f16 = src1_data.f16 - src2_data.f16; - break; // assert(0); break; - case F32_TYPE: - data.f32 = src1_data.f32 - src2_data.f32; - break; - case F64_TYPE: - case FF64_TYPE: - data.f64 = src1_data.f64 - src2_data.f64; - break; - default: - assert(0); - break; - } + case B64_TYPE: + case U64_TYPE: + data.u64 = src1_data.u64 - src2_data.u64; break; + case F16_TYPE: data.f16 = src1_data.f16 - src2_data.f16; break;//assert(0); break; + case F32_TYPE: data.f32 = src1_data.f32 - src2_data.f32; break; + case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 - src2_data.f64; break; + default: assert(0); break; + } - thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); + thread->set_operand_value(dst,data, i_type, thread, pI, overflow, carry); } -void nop_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - // Do nothing +void nop_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + // Do nothing } -void subc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void suld_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void sured_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void sust_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void suq_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} +void subc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void suld_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void sured_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void sust_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void suq_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } + union intfloat { - int a; - float b; + int a; + float b; }; -float reduce_precision(float x, unsigned bits) { - intfloat tmp; - tmp.b = x; - int v = tmp.a; - int man = v & ((1 << 23) - 1); - int mask = ((1 << bits) - 1) << (23 - bits); - int nv = (v & ((-1) - ((1 << 23) - 1))) | (mask & man); - tmp.a = nv; - float result = tmp.b; - return result; -} - -unsigned wrap(unsigned x, unsigned y, unsigned mx, unsigned my, - size_t elem_size) { - unsigned nx = (mx + x) % mx; - unsigned ny = (my + y) % my; - return nx + mx * ny; -} - -unsigned clamp(unsigned x, unsigned y, unsigned mx, unsigned my, - size_t elem_size) { - unsigned nx = x; - while (nx >= mx) nx -= elem_size; - unsigned ny = (y >= my) ? my - 1 : y; - return nx + mx * ny; -} - -typedef unsigned (*texAddr_t)(unsigned x, unsigned y, unsigned mx, unsigned my, - size_t elem_size); -float tex_linf_sampling(memory_space *mem, unsigned tex_array_base, int x, - int y, unsigned int width, unsigned int height, - size_t elem_size, float alpha, float beta, - texAddr_t b_lim) { - float Tij; - float Ti1j; - float Tij1; - float Ti1j1; - - mem->read(tex_array_base + b_lim(x, y, width, height, elem_size), 4, &Tij); - mem->read(tex_array_base + b_lim(x + elem_size, y, width, height, elem_size), - 4, &Ti1j); - mem->read(tex_array_base + b_lim(x, y + 1, width, height, elem_size), 4, - &Tij1); - mem->read( - tex_array_base + b_lim(x + elem_size, y + 1, width, height, elem_size), 4, - &Ti1j1); - - float sample = (1 - alpha) * (1 - beta) * Tij + alpha * (1 - beta) * Ti1j + - (1 - alpha) * beta * Tij1 + alpha * beta * Ti1j1; - - return sample; -} - -float textureNormalizeElementSigned(int element, int bits) { - if (bits) { - int maxN = (1 << bits) - 1; - // removing upper bits - element &= maxN; - // normalizing the number to [-1.0,1.0] - maxN >>= 1; - float output = (float)element / maxN; - if (output < -1.0f) output = -1.0f; - return output; - } else { - return 0.0f; - } -} - -float textureNormalizeElementUnsigned(unsigned int element, int bits) { - if (bits) { - unsigned int maxN = (1 << bits) - 1; - // removing upper bits and normalizing the number to [0.0,1.0] - return (float)(element & maxN) / maxN; - } else { - return 0.0f; - } -} - -void textureNormalizeOutput(const struct cudaChannelFormatDesc &desc, - ptx_reg_t &datax, ptx_reg_t &datay, - ptx_reg_t &dataz, ptx_reg_t &dataw) { - if (desc.f == cudaChannelFormatKindSigned) { - datax.f32 = textureNormalizeElementSigned(datax.s32, desc.x); - datay.f32 = textureNormalizeElementSigned(datay.s32, desc.y); - dataz.f32 = textureNormalizeElementSigned(dataz.s32, desc.z); - dataw.f32 = textureNormalizeElementSigned(dataw.s32, desc.w); - } else if (desc.f == cudaChannelFormatKindUnsigned) { - datax.f32 = textureNormalizeElementUnsigned(datax.u32, desc.x); - datay.f32 = textureNormalizeElementUnsigned(datay.u32, desc.y); - dataz.f32 = textureNormalizeElementUnsigned(dataz.u32, desc.z); - dataw.f32 = textureNormalizeElementUnsigned(dataw.u32, desc.w); - } else { - assert(0 && - "Undefined texture read mode: cudaReadModeNormalizedFloat expect " - "integer elements"); - } -} - -void tex_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - unsigned dimension = pI->dimension(); - const operand_info &dst = - pI->dst(); // the registers to which fetched texel will be placed - const operand_info &src1 = pI->src1(); // the name of the texture - const operand_info &src2 = pI->src2(); // the vector registers containing - // coordinates of the texel to be - // fetched - - std::string texname = src1.name(); - unsigned to_type = pI->get_type(); - unsigned c_type = pI->get_type2(); - fflush(stdout); - ptx_reg_t data1, data2, data3, data4; - if (!thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs) - thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs = new ptx_reg_t[4]; - unsigned nelem = src2.get_vect_nelem(); - thread->get_vector_operand_values( - src2, thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs, - nelem); // ptx_reg should be 4 entry vector type...coordinates into - // texture - /* - For programs with many streams, textures can be bound and unbound - asynchronously. This means we need to use the kernel's "snapshot" of - the state of the texture mappings when it was launched (so that we - don't try to access the incorrect texture mapping if it's been updated, - or that we don't access a mapping that has been unbound). - */ - gpgpu_t *gpu = thread->get_gpu(); - kernel_info_t &k = thread->get_kernel(); - const struct textureReference *texref = gpu->get_texref(texname); - const struct cudaArray *cuArray = k.get_texarray(texname); - const struct textureInfo *texInfo = k.get_texinfo(texname); - const struct textureReferenceAttr *texAttr = gpu->get_texattr(texname); - - // assume always 2D f32 input - // access array with src2 coordinates - memory_space *mem = thread->get_global_memory(); - float x_f32, y_f32; - size_t size; - int t; - unsigned tex_array_base; - unsigned int width = 0, height = 0; - int x = 0; - int y = 0; - unsigned tex_array_index; - float alpha = 0, beta = 0; - - type_info_key::type_decode(to_type, size, t); - tex_array_base = cuArray->devPtr32; - - switch (dimension) { - case GEOM_MODIFIER_1D: +float reduce_precision( float x, unsigned bits ) +{ + intfloat tmp; + tmp.b = x; + int v = tmp.a; + int man = v & ((1<<23)-1); + int mask = ((1<= mx) nx -= elem_size; + unsigned ny = (y >= my)? my - 1 : y; + return nx + mx*ny; +} + +typedef unsigned (*texAddr_t) (unsigned x, unsigned y, unsigned mx, unsigned my, size_t elem_size); +float tex_linf_sampling(memory_space* mem, unsigned tex_array_base, + int x, int y, unsigned int width, unsigned int height, size_t elem_size, + float alpha, float beta, texAddr_t b_lim) +{ + float Tij; + float Ti1j; + float Tij1; + float Ti1j1; + + mem->read(tex_array_base + b_lim(x,y,width,height,elem_size), 4, &Tij); + mem->read(tex_array_base + b_lim(x+elem_size,y,width,height,elem_size), 4, &Ti1j); + mem->read(tex_array_base + b_lim(x,y+1,width,height,elem_size), 4, &Tij1); + mem->read(tex_array_base + b_lim(x+elem_size,y+1,width,height,elem_size), 4, &Ti1j1); + + float sample = (1-alpha)*(1-beta)*Tij + + alpha*(1-beta)*Ti1j + + (1-alpha)*beta*Tij1 + + alpha*beta*Ti1j1; + + return sample; +} + +float textureNormalizeElementSigned(int element, int bits) +{ + if (bits) { + int maxN = (1 << bits) - 1; + // removing upper bits + element &= maxN; + // normalizing the number to [-1.0,1.0] + maxN >>= 1; + float output = (float) element / maxN; + if (output < -1.0f) output = -1.0f; + return output; + } else { + return 0.0f; + } +} + +float textureNormalizeElementUnsigned(unsigned int element, int bits) +{ + if (bits) { + unsigned int maxN = (1 << bits) - 1; + // removing upper bits and normalizing the number to [0.0,1.0] + return (float)(element & maxN) / maxN; + } else { + return 0.0f; + } +} + +void textureNormalizeOutput( const struct cudaChannelFormatDesc& desc, ptx_reg_t& datax, ptx_reg_t& datay, ptx_reg_t& dataz, ptx_reg_t& dataw ) +{ + if (desc.f == cudaChannelFormatKindSigned) { + datax.f32 = textureNormalizeElementSigned( datax.s32, desc.x ); + datay.f32 = textureNormalizeElementSigned( datay.s32, desc.y ); + dataz.f32 = textureNormalizeElementSigned( dataz.s32, desc.z ); + dataw.f32 = textureNormalizeElementSigned( dataw.s32, desc.w ); + } else if (desc.f == cudaChannelFormatKindUnsigned) { + datax.f32 = textureNormalizeElementUnsigned( datax.u32, desc.x ); + datay.f32 = textureNormalizeElementUnsigned( datay.u32, desc.y ); + dataz.f32 = textureNormalizeElementUnsigned( dataz.u32, desc.z ); + dataw.f32 = textureNormalizeElementUnsigned( dataw.u32, desc.w ); + } else { + assert(0 && "Undefined texture read mode: cudaReadModeNormalizedFloat expect integer elements"); + } +} + +void tex_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + unsigned dimension = pI->dimension(); + const operand_info &dst = pI->dst(); //the registers to which fetched texel will be placed + const operand_info &src1 = pI->src1(); //the name of the texture + const operand_info &src2 = pI->src2(); //the vector registers containing coordinates of the texel to be fetched + + std::string texname = src1.name(); + unsigned to_type = pI->get_type(); + unsigned c_type = pI->get_type2(); + fflush(stdout); + ptx_reg_t data1, data2, data3, data4; + if (!thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs) + thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs = new ptx_reg_t[4]; + unsigned nelem = src2.get_vect_nelem(); + thread->get_vector_operand_values(src2, thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs, nelem); //ptx_reg should be 4 entry vector type...coordinates into texture + /* + For programs with many streams, textures can be bound and unbound + asynchronously. This means we need to use the kernel's "snapshot" of + the state of the texture mappings when it was launched (so that we + don't try to access the incorrect texture mapping if it's been updated, + or that we don't access a mapping that has been unbound). + */ + gpgpu_t *gpu = thread->get_gpu(); + kernel_info_t &k = thread->get_kernel(); + const struct textureReference* texref = gpu->get_texref(texname); + const struct cudaArray* cuArray = k.get_texarray(texname); + const struct textureInfo* texInfo = k.get_texinfo(texname); + const struct textureReferenceAttr* texAttr = gpu->get_texattr(texname); + + //assume always 2D f32 input + //access array with src2 coordinates + memory_space *mem = thread->get_global_memory(); + float x_f32, y_f32; + size_t size; + int t; + unsigned tex_array_base; + unsigned int width = 0, height = 0; + int x = 0; + int y = 0; + unsigned tex_array_index; + float alpha=0, beta=0; + + type_info_key::type_decode(to_type,size,t); + tex_array_base = cuArray->devPtr32; + + switch (dimension) { + case GEOM_MODIFIER_1D: width = cuArray->width; height = cuArray->height; if (texref->normalized) { - assert(c_type == F32_TYPE); - x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - if (texref->addressMode[0] == cudaAddressModeClamp) { - x_f32 = (x_f32 > 1.0) ? 1.0 : x_f32; - x_f32 = (x_f32 < 0.0) ? 0.0 : x_f32; - } else if (texref->addressMode[0] == cudaAddressModeWrap) { - x_f32 = x_f32 - floor(x_f32); - } - - if (texref->filterMode == cudaFilterModeLinear) { - float xb = x_f32 * width - 0.5; - alpha = xb - floor(xb); - alpha = reduce_precision(alpha, 9); - beta = 0.0; - - x = (int)floor(xb); - y = 0; - } else { - x = (int)floor(x_f32 * width); - y = 0; - } + assert(c_type == F32_TYPE); + x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; + if (texref->addressMode[0] == cudaAddressModeClamp) { + x_f32 = (x_f32 > 1.0)? 1.0 : x_f32; + x_f32 = (x_f32 < 0.0)? 0.0 : x_f32; + } else if (texref->addressMode[0] == cudaAddressModeWrap) { + x_f32 = x_f32 - floor(x_f32); + } + + if( texref->filterMode == cudaFilterModeLinear ) { + float xb = x_f32 * width - 0.5; + alpha = xb - floor(xb); + alpha = reduce_precision(alpha,9); + beta = 0.0; + + x = (int)floor(xb); + y = 0; + } else { + x = (int) floor(x_f32 * width); + y = 0; + } } else { - switch (c_type) { - case S32_TYPE: + switch ( c_type ) { + case S32_TYPE: x = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].s32; - assert(texref->filterMode == cudaFilterModePoint); - break; - case F32_TYPE: + assert(texref->filterMode == cudaFilterModePoint); + break; + case F32_TYPE: x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - alpha = x_f32 - - floor(x_f32); // offset into subtexel (for linear sampling) - x = (int)x_f32; - break; - default: - assert(0 && "Unsupported texture coordinate type."); - } - // handle texture fetch that exceeded boundaries - if (texref->addressMode[0] == cudaAddressModeClamp) { - x = (x > width - 1) ? (width - 1) : x; - x = (x < 0) ? 0 : x; - } else if (texref->addressMode[0] == cudaAddressModeWrap) { - x = x % width; - } + alpha = x_f32 - floor(x_f32); // offset into subtexel (for linear sampling) + x = (int) x_f32; + break; + default: assert(0 && "Unsupported texture coordinate type."); + } + // handle texture fetch that exceeded boundaries + if (texref->addressMode[0] == cudaAddressModeClamp) { + x = (x > width - 1)? (width - 1) : x; + x = (x < 0)? 0 : x; + } else if (texref->addressMode[0] == cudaAddressModeWrap) { + x = x % width; + } } - width *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + - cuArray->desc.z) / - 8; - x *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + - cuArray->desc.z) / - 8; + width *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; + x *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; tex_array_index = tex_array_base + x; break; - case GEOM_MODIFIER_2D: + case GEOM_MODIFIER_2D: width = cuArray->width; height = cuArray->height; if (texref->normalized) { - x_f32 = reduce_precision( - thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32, 16); - y_f32 = reduce_precision( - thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32, 15); - - if (texref->addressMode[0]) { // clamp - if (x_f32 < 0) x_f32 = 0; - if (x_f32 >= 1) x_f32 = 1 - 1 / x_f32; - } else { // wrap - x_f32 = x_f32 - floor(x_f32); - } - if (texref->addressMode[1]) { // clamp - if (y_f32 < 0) y_f32 = 0; - if (y_f32 >= 1) y_f32 = 1 - 1 / y_f32; - } else { // wrap - y_f32 = y_f32 - floor(y_f32); - } - - if (texref->filterMode == cudaFilterModeLinear) { - float xb = x_f32 * width - 0.5; - float yb = y_f32 * height - 0.5; - alpha = xb - floor(xb); - beta = yb - floor(yb); - alpha = reduce_precision(alpha, 9); - beta = reduce_precision(beta, 9); - - x = (int)floor(xb); - y = (int)floor(yb); - } else { - x = (int)floor(x_f32 * width); - y = (int)floor(y_f32 * height); - } + x_f32 = reduce_precision(thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32,16); + y_f32 = reduce_precision(thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32,15); + + if (texref->addressMode[0]) {//clamp + if (x_f32<0) x_f32 = 0; + if (x_f32>=1) x_f32 = 1 - 1/x_f32; + } else {//wrap + x_f32 = x_f32 - floor(x_f32); + } + if (texref->addressMode[1]) {//clamp + if (y_f32<0) y_f32 = 0; + if (y_f32>=1) y_f32 = 1 - 1/y_f32; + } else {//wrap + y_f32 = y_f32 - floor(y_f32); + } + + if( texref->filterMode == cudaFilterModeLinear ) { + float xb = x_f32 * width - 0.5; + float yb = y_f32 * height - 0.5; + alpha = xb - floor(xb); + beta = yb - floor(yb); + alpha = reduce_precision(alpha,9); + beta = reduce_precision(beta,9); + + x = (int)floor(xb); + y = (int)floor(yb); + } else { + x = (int) floor(x_f32 * width); + y = (int) floor(y_f32 * height); + } } else { - x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - y_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32; - - alpha = x_f32 - floor(x_f32); - beta = y_f32 - floor(y_f32); - - x = (int)x_f32; - y = (int)y_f32; - if (texref->addressMode[0]) { // clamp - if (x < 0) x = 0; - if (x >= (int)width) x = width - 1; - } else { // wrap - x = x % width; - if (x < 0) x *= -1; - } - if (texref->addressMode[1]) { // clamp - if (y < 0) y = 0; - if (y >= (int)height) y = height - 1; - } else { // wrap - y = y % height; - if (y < 0) y *= -1; - } + x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; + y_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32; + + alpha = x_f32 - floor(x_f32); + beta = y_f32 - floor(y_f32); + + x = (int) x_f32; + y = (int) y_f32; + if (texref->addressMode[0]) {//clamp + if (x<0) x = 0; + if (x>= (int)width) x = width-1; + } else {//wrap + x = x % width; + if (x < 0) x*= -1; + } + if (texref->addressMode[1]) {//clamp + if (y<0) y = 0; + if (y>= (int)height) y = height -1; + } else {//wrap + y = y % height; + if (y < 0) y *= -1; + } } - width *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + - cuArray->desc.z) / - 8; - x *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + - cuArray->desc.z) / - 8; - tex_array_index = tex_array_base + (x + width * y); - break; - default: - assert(0); - break; - } - switch (to_type) { - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case B8_TYPE: - case B16_TYPE: - case B32_TYPE: - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: { - unsigned long long elementOffset = 0; // offset into the next element - mem->read(tex_array_index, cuArray->desc.x / 8, &data1.u32); - elementOffset += cuArray->desc.x / 8; + width *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; + x *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; + tex_array_index = tex_array_base + (x + width*y); + break; + default: + assert(0); break; + } + switch ( to_type ) { + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case B8_TYPE: + case B16_TYPE: + case B32_TYPE: + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: { + unsigned long long elementOffset = 0; // offset into the next element + mem->read( tex_array_index, cuArray->desc.x/8, &data1.u32); + elementOffset += cuArray->desc.x/8; if (cuArray->desc.y) { - mem->read(tex_array_index + elementOffset, cuArray->desc.y / 8, - &data2.u32); - elementOffset += cuArray->desc.y / 8; - if (cuArray->desc.z) { - mem->read(tex_array_index + elementOffset, cuArray->desc.z / 8, - &data3.u32); - elementOffset += cuArray->desc.z / 8; - if (cuArray->desc.w) - mem->read(tex_array_index + elementOffset, cuArray->desc.w / 8, - &data4.u32); - } + mem->read( tex_array_index + elementOffset, cuArray->desc.y/8, &data2.u32); + elementOffset += cuArray->desc.y/8; + if (cuArray->desc.z) { + mem->read( tex_array_index + elementOffset, cuArray->desc.z/8, &data3.u32); + elementOffset += cuArray->desc.z/8; + if (cuArray->desc.w) + mem->read( tex_array_index + elementOffset, cuArray->desc.w/8, &data4.u32); + } } break; - } - case B64_TYPE: - case U64_TYPE: - case S64_TYPE: - mem->read(tex_array_index, 8, &data1.u64); + } + case B64_TYPE: + case U64_TYPE: + case S64_TYPE: + mem->read( tex_array_index, 8, &data1.u64); if (cuArray->desc.y) { - mem->read(tex_array_index + 8, 8, &data2.u64); - if (cuArray->desc.z) { - mem->read(tex_array_index + 16, 8, &data3.u64); - if (cuArray->desc.w) mem->read(tex_array_index + 24, 8, &data4.u64); - } + mem->read( tex_array_index+8, 8, &data2.u64); + if (cuArray->desc.z) { + mem->read( tex_array_index+16, 8, &data3.u64); + if (cuArray->desc.w) + mem->read( tex_array_index+24, 8, &data4.u64); + } } break; - case F16_TYPE: - assert(0); - break; - case F32_TYPE: { - if (texref->filterMode == cudaFilterModeLinear) { - texAddr_t b_lim = wrap; - if (texref->addressMode[0] == cudaAddressModeClamp) { - b_lim = clamp; - } - size_t elem_size = (cuArray->desc.x + cuArray->desc.y + - cuArray->desc.z + cuArray->desc.w) / - 8; - size_t elem_ofst = 0; - - data1.f32 = - tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, - height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.x / 8; - if (cuArray->desc.y) { - data2.f32 = - tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, - height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.y / 8; - if (cuArray->desc.z) { - data3.f32 = - tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, - height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.z / 8; - if (cuArray->desc.w) - data4.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, - y, width, height, elem_size, alpha, - beta, b_lim); - } - } + case F16_TYPE: assert(0); break; + case F32_TYPE: { + if( texref->filterMode == cudaFilterModeLinear ) { + texAddr_t b_lim = wrap; + if ( texref->addressMode[0] == cudaAddressModeClamp ) { + b_lim = clamp; + } + size_t elem_size = (cuArray->desc.x + cuArray->desc.y + cuArray->desc.z + cuArray->desc.w) / 8; + size_t elem_ofst = 0; + + data1.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.x / 8; + if (cuArray->desc.y) { + data2.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.y / 8; + if (cuArray->desc.z) { + data3.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.z / 8; + if (cuArray->desc.w) + data4.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); + } + } } else { - mem->read(tex_array_index, cuArray->desc.x / 8, &data1.f32); - if (cuArray->desc.y) { - mem->read(tex_array_index + 4, cuArray->desc.y / 8, &data2.f32); - if (cuArray->desc.z) { - mem->read(tex_array_index + 8, cuArray->desc.z / 8, &data3.f32); - if (cuArray->desc.w) - mem->read(tex_array_index + 12, cuArray->desc.w / 8, &data4.f32); - } - } + mem->read( tex_array_index, cuArray->desc.x/8, &data1.f32); + if (cuArray->desc.y) { + mem->read( tex_array_index+4, cuArray->desc.y/8, &data2.f32); + if (cuArray->desc.z) { + mem->read( tex_array_index+8, cuArray->desc.z/8, &data3.f32); + if (cuArray->desc.w) + mem->read( tex_array_index+12, cuArray->desc.w/8, &data4.f32); + } + } } - } break; - case F64_TYPE: - case FF64_TYPE: - mem->read(tex_array_index, 8, &data1.f64); + } break; + case F64_TYPE: + case FF64_TYPE: + mem->read( tex_array_index, 8, &data1.f64); if (cuArray->desc.y) { - mem->read(tex_array_index + 8, 8, &data2.f64); - if (cuArray->desc.z) { - mem->read(tex_array_index + 16, 8, &data3.f64); - if (cuArray->desc.w) mem->read(tex_array_index + 24, 8, &data4.f64); - } + mem->read( tex_array_index+8, 8, &data2.f64); + if (cuArray->desc.z) { + mem->read( tex_array_index+16, 8, &data3.f64); + if (cuArray->desc.w) + mem->read( tex_array_index+24, 8, &data4.f64); + } } break; - default: - assert(0); - break; - } - int x_block_coord, y_block_coord, memreqindex, blockoffset; + default: assert(0); break; + } + int x_block_coord, y_block_coord, memreqindex, blockoffset; - switch (dimension) { - case GEOM_MODIFIER_1D: + switch (dimension) { + case GEOM_MODIFIER_1D: thread->m_last_effective_address = tex_array_index; break; - case GEOM_MODIFIER_2D: + case GEOM_MODIFIER_2D: x_block_coord = x >> (texInfo->Tx_numbits + texInfo->texel_size_numbits); y_block_coord = y >> texInfo->Ty_numbits; - memreqindex = - ((y_block_coord * cuArray->width / texInfo->Tx) + x_block_coord) << 6; + memreqindex = ((y_block_coord*cuArray->width/texInfo->Tx)+x_block_coord)<<6; - blockoffset = (x % (texInfo->Tx * texInfo->texel_size) + - (y % (texInfo->Ty) - << (texInfo->Tx_numbits + texInfo->texel_size_numbits))); + blockoffset = (x%(texInfo->Tx*texInfo->texel_size) + (y%(texInfo->Ty)<<(texInfo->Tx_numbits + texInfo->texel_size_numbits))); memreqindex += blockoffset; - thread->m_last_effective_address = - tex_array_base + memreqindex; // tex_array_index; + thread->m_last_effective_address = tex_array_base + memreqindex;//tex_array_index; break; - default: + default: assert(0); - } - thread->m_last_memory_space = tex_space; - - // normalize output into floating point numbers according to the texture read - // mode - if (texAttr->m_readmode == cudaReadModeNormalizedFloat) { - textureNormalizeOutput(cuArray->desc, data1, data2, data3, data4); - } else { - assert(texAttr->m_readmode == cudaReadModeElementType); - } - - thread->set_vector_operand_values(dst, data1, data2, data3, data4); -} - -void txq_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void trap_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vabsdiff_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vadd_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vmad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vmax_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vmin_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vset_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vshl_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vshr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vsub_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} - -void vote_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - static bool first_in_warp = true; - static bool and_all; - static bool or_all; - static unsigned int ballot_result; - static std::list threads_in_warp; - static unsigned last_tid; - - if (first_in_warp) { - first_in_warp = false; - threads_in_warp.clear(); - and_all = true; - or_all = false; - ballot_result = 0; - int offset = 31; - while ((offset >= 0) && !pI->active(offset)) offset--; - assert(offset >= 0); - last_tid = - (thread->get_hw_tid() - (thread->get_hw_tid() % pI->warp_size())) + - offset; - } - - ptx_reg_t src1_data; - const operand_info &src1 = pI->src1(); - src1_data = thread->get_operand_value(src1, pI->dst(), PRED_TYPE, thread, 1); - - // predicate value was changed so the lowest bit being set means the zero flag - // is set. - // As a result, the value of src1_data.pred must be inverted to get proper - // behavior - bool pred_value = !(src1_data.pred & 0x0001); - bool invert = src1.is_neg_pred(); - - threads_in_warp.push_back(thread); - and_all &= (invert ^ pred_value); - or_all |= (invert ^ pred_value); - - // vote.ballot - if (invert ^ pred_value) { - int lane_id = thread->get_hw_tid() % pI->warp_size(); - ballot_result |= (1 << lane_id); - } + } + thread->m_last_memory_space = tex_space; + + // normalize output into floating point numbers according to the texture read mode + if (texAttr->m_readmode == cudaReadModeNormalizedFloat) { + textureNormalizeOutput(cuArray->desc, data1, data2, data3, data4); + } else { + assert(texAttr->m_readmode == cudaReadModeElementType); + } + + thread->set_vector_operand_values(dst,data1,data2,data3,data4); +} + +void txq_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void trap_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vabsdiff_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vadd_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vmad_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vmax_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vmin_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vset_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vshl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vshr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vsub_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } + +void vote_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + static bool first_in_warp = true; + static bool and_all; + static bool or_all; + static unsigned int ballot_result; + static std::list threads_in_warp; + static unsigned last_tid; + + if( first_in_warp ) { + first_in_warp = false; + threads_in_warp.clear(); + and_all = true; + or_all = false; + ballot_result = 0; + int offset=31; + while( (offset>=0) && !pI->active(offset) ) + offset--; + assert( offset >= 0 ); + last_tid = (thread->get_hw_tid() - (thread->get_hw_tid()%pI->warp_size())) + offset; + } + + ptx_reg_t src1_data; + const operand_info &src1 = pI->src1(); + src1_data = thread->get_operand_value(src1, pI->dst(), PRED_TYPE, thread, 1); + + //predicate value was changed so the lowest bit being set means the zero flag is set. + //As a result, the value of src1_data.pred must be inverted to get proper behavior + bool pred_value = !(src1_data.pred & 0x0001); + bool invert = src1.is_neg_pred(); + + threads_in_warp.push_back(thread); + and_all &= (invert ^ pred_value); + or_all |= (invert ^ pred_value); + + // vote.ballot + if (invert ^ pred_value) { + int lane_id = thread->get_hw_tid() % pI->warp_size(); + ballot_result |= (1 << lane_id); + } + + if( thread->get_hw_tid() == last_tid ) { + if (pI->vote_mode() == ptx_instruction::vote_ballot) { + ptx_reg_t data = ballot_result; + for( std::list::iterator t=threads_in_warp.begin(); t!=threads_in_warp.end(); ++t ) { + const operand_info &dst = pI->dst(); + (*t)->set_operand_value(dst,data, pI->get_type(), (*t), pI); + } + } else { + bool pred_value = false; - if (thread->get_hw_tid() == last_tid) { - if (pI->vote_mode() == ptx_instruction::vote_ballot) { - ptx_reg_t data = ballot_result; - for (std::list::iterator t = threads_in_warp.begin(); - t != threads_in_warp.end(); ++t) { - const operand_info &dst = pI->dst(); - (*t)->set_operand_value(dst, data, pI->get_type(), (*t), pI); - } - } else { - bool pred_value = false; - - switch (pI->vote_mode()) { - case ptx_instruction::vote_any: - pred_value = or_all; - break; - case ptx_instruction::vote_all: - pred_value = and_all; - break; - case ptx_instruction::vote_uni: - pred_value = (or_all ^ and_all); - break; - default: - abort(); - } - ptx_reg_t data; - data.pred = pred_value ? 0 : 1; // the way ptxplus handles the zero flag, - // 1 = false and 0 = true - - for (std::list::iterator t = threads_in_warp.begin(); - t != threads_in_warp.end(); ++t) { - const operand_info &dst = pI->dst(); - (*t)->set_operand_value(dst, data, PRED_TYPE, (*t), pI); + switch( pI->vote_mode() ) { + case ptx_instruction::vote_any: pred_value = or_all; break; + case ptx_instruction::vote_all: pred_value = and_all; break; + case ptx_instruction::vote_uni: pred_value = (or_all ^ and_all); break; + default: + abort(); + } + ptx_reg_t data; + data.pred = pred_value?0:1; //the way ptxplus handles the zero flag, 1 = false and 0 = true + + for( std::list::iterator t=threads_in_warp.begin(); t!=threads_in_warp.end(); ++t ) { + const operand_info &dst = pI->dst(); + (*t)->set_operand_value(dst,data, PRED_TYPE, (*t), pI); + } } - } - first_in_warp = true; - } -} - -void xor_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - - // the way ptxplus handles predicates: 1 = false and 0 = true - if (i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) ^ ~(src2_data.pred)); - else - data.u64 = src1_data.u64 ^ src2_data.u64; + first_in_warp = true; + } +} + +void xor_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) ^ ~(src2_data.pred)); + else + data.u64 = src1_data.u64 ^ src2_data.u64; + + thread->set_operand_value(dst,data, i_type, thread, pI); +} + +void inst_not_implemented( const ptx_instruction * pI ) +{ + printf("GPGPU-Sim PTX: ERROR (%s:%u) instruction \"%s\" not (yet) implemented\n", + pI->source_file(), + pI->source_line(), + pI->get_opcode_cstr() ); + abort(); +} + +ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread) +{ + ptx_reg_t result; + memory_space *mem = NULL; + size_t size; + int t; + result.u64=0; + + //complete other cases for reading from memory, such as reading from other const memory + if(opInfo.get_addr_space() == global_space) + { + mem = thread->get_global_memory(); + type_info_key::type_decode(type,size,t); + mem->read(opData.u32,size/8,&result.u64); + if( type == S16_TYPE || type == S32_TYPE ) + sign_extend(result,size,dstInfo); + } + else if(opInfo.get_addr_space() == shared_space) + { + mem = thread->m_shared_mem; + type_info_key::type_decode(type,size,t); + mem->read(opData.u32,size/8,&result.u64); + + if( type == S16_TYPE || type == S32_TYPE ) + sign_extend(result,size,dstInfo); + + } + else if(opInfo.get_addr_space() == const_space) + { + mem = thread->get_global_memory(); + type_info_key::type_decode(type,size,t); + + mem->read((opData.u32 + opInfo.get_const_mem_offset()),size/8,&result.u64); + + if( type == S16_TYPE || type == S32_TYPE ) + sign_extend(result,size,dstInfo); + } + else + { + result = opData; + } + + if(opInfo.get_operand_lohi() == 1) + { + result.u64 = result.u64 & 0xFFFF; + } + else if(opInfo.get_operand_lohi() == 2) + { + result.u64 = (result.u64>>16) & 0xFFFF; + } - thread->set_operand_value(dst, data, i_type, thread, pI); -} + if(opInfo.get_operand_neg() == true) { + result.f32 = -result.f32; + } -void inst_not_implemented(const ptx_instruction *pI) { - printf( - "GPGPU-Sim PTX: ERROR (%s:%u) instruction \"%s\" not (yet) implemented\n", - pI->source_file(), pI->source_line(), pI->get_opcode_cstr()); - abort(); + return result; } -ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, - operand_info dstInfo, unsigned type, - ptx_thread_info *thread) { - ptx_reg_t result; - memory_space *mem = NULL; - size_t size; - int t; - result.u64 = 0; - - // complete other cases for reading from memory, such as reading from other - // const memory - if (opInfo.get_addr_space() == global_space) { - mem = thread->get_global_memory(); - type_info_key::type_decode(type, size, t); - mem->read(opData.u32, size / 8, &result.u64); - if (type == S16_TYPE || type == S32_TYPE) - sign_extend(result, size, dstInfo); - } else if (opInfo.get_addr_space() == shared_space) { - mem = thread->m_shared_mem; - type_info_key::type_decode(type, size, t); - mem->read(opData.u32, size / 8, &result.u64); - - if (type == S16_TYPE || type == S32_TYPE) - sign_extend(result, size, dstInfo); - - } else if (opInfo.get_addr_space() == const_space) { - mem = thread->get_global_memory(); - type_info_key::type_decode(type, size, t); - - mem->read((opData.u32 + opInfo.get_const_mem_offset()), size / 8, - &result.u64); - - if (type == S16_TYPE || type == S32_TYPE) - sign_extend(result, size, dstInfo); - } else { - result = opData; - } - - if (opInfo.get_operand_lohi() == 1) { - result.u64 = result.u64 & 0xFFFF; - } else if (opInfo.get_operand_lohi() == 2) { - result.u64 = (result.u64 >> 16) & 0xFFFF; - } - - if (opInfo.get_operand_neg() == true) { - result.f32 = -result.f32; - } - - return result; -} diff --git a/src/cuda-sim/memory.cc b/src/cuda-sim/memory.cc index b630da9..4b2acdf 100644 --- a/src/cuda-sim/memory.cc +++ b/src/cuda-sim/memory.cc @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -29,206 +27,190 @@ #include "memory.h" #include -#include "../../libcuda/gpgpu_context.h" #include "../debug.h" +#include "../../libcuda/gpgpu_context.h" + +template memory_space_impl::memory_space_impl( std::string name, unsigned hash_size ) +{ + m_name = name; + MEM_MAP_RESIZE(hash_size); -template -memory_space_impl::memory_space_impl(std::string name, - unsigned hash_size) { - m_name = name; - MEM_MAP_RESIZE(hash_size); - - m_log2_block_size = -1; - for (unsigned n = 0, mask = 1; mask != 0; mask <<= 1, n++) { - if (BSIZE & mask) { - assert(m_log2_block_size == (unsigned)-1); - m_log2_block_size = n; - } - } - assert(m_log2_block_size != (unsigned)-1); + m_log2_block_size = -1; + for( unsigned n=0, mask=1; mask != 0; mask <<= 1, n++ ) { + if( BSIZE & mask ) { + assert( m_log2_block_size == (unsigned)-1 ); + m_log2_block_size = n; + } + } + assert( m_log2_block_size != (unsigned)-1 ); } -template -void memory_space_impl::write_only(mem_addr_t offset, mem_addr_t index, - size_t length, const void *data) { - m_data[index].write(offset, length, (const unsigned char *)data); +template void memory_space_impl::write_only( mem_addr_t offset, mem_addr_t index, size_t length, const void *data) +{ + m_data[index].write(offset,length,(const unsigned char*)data); } -template -void memory_space_impl::write(mem_addr_t addr, size_t length, - const void *data, - class ptx_thread_info *thd, - const ptx_instruction *pI) { - mem_addr_t index = addr >> m_log2_block_size; - - if ((addr + length) <= (index + 1) * BSIZE) { - // fast route for intra-block access - unsigned offset = addr & (BSIZE - 1); - unsigned nbytes = length; - m_data[index].write(offset, nbytes, (const unsigned char *)data); - } else { - // slow route for inter-block access - unsigned nbytes_remain = length; - unsigned src_offset = 0; - mem_addr_t current_addr = addr; - - while (nbytes_remain > 0) { - unsigned offset = current_addr & (BSIZE - 1); - mem_addr_t page = current_addr >> m_log2_block_size; - mem_addr_t access_limit = offset + nbytes_remain; - if (access_limit > BSIZE) { - access_limit = BSIZE; +template void memory_space_impl::write( mem_addr_t addr, size_t length, const void *data, class ptx_thread_info *thd, const ptx_instruction *pI) +{ + + mem_addr_t index = addr >> m_log2_block_size; + + if ( (addr+length) <= (index+1)*BSIZE ) { + // fast route for intra-block access + unsigned offset = addr & (BSIZE-1); + unsigned nbytes = length; + m_data[index].write(offset,nbytes,(const unsigned char*)data); + } else { + // slow route for inter-block access + unsigned nbytes_remain = length; + unsigned src_offset = 0; + mem_addr_t current_addr = addr; + + while (nbytes_remain > 0) { + unsigned offset = current_addr & (BSIZE-1); + mem_addr_t page = current_addr >> m_log2_block_size; + mem_addr_t access_limit = offset + nbytes_remain; + if (access_limit > BSIZE) { + access_limit = BSIZE; + } + + size_t tx_bytes = access_limit - offset; + m_data[page].write(offset, tx_bytes, &((const unsigned char*)data)[src_offset]); + + // advance pointers + src_offset += tx_bytes; + current_addr += tx_bytes; + nbytes_remain -= tx_bytes; } - - size_t tx_bytes = access_limit - offset; - m_data[page].write(offset, tx_bytes, - &((const unsigned char *)data)[src_offset]); - - // advance pointers - src_offset += tx_bytes; - current_addr += tx_bytes; - nbytes_remain -= tx_bytes; - } - assert(nbytes_remain == 0); - } - if (!m_watchpoints.empty()) { - std::map::iterator i; - for (i = m_watchpoints.begin(); i != m_watchpoints.end(); i++) { - mem_addr_t wa = i->second; - if (((addr <= wa) && ((addr + length) > wa)) || - ((addr > wa) && (addr < (wa + 4)))) - thd->get_gpu()->gpgpu_ctx->the_gpgpusim->g_the_gpu->hit_watchpoint( - i->first, thd, pI); - } - } + assert(nbytes_remain == 0); + } + if( !m_watchpoints.empty() ) { + std::map::iterator i; + for( i=m_watchpoints.begin(); i!=m_watchpoints.end(); i++ ) { + mem_addr_t wa = i->second; + if( ((addr<=wa) && ((addr+length)>wa)) || ((addr>wa) && (addr < (wa+4))) ) + thd->get_gpu()->gpgpu_ctx->the_gpgpusim->g_the_gpu->hit_watchpoint(i->first,thd,pI); + } + } } -template -void memory_space_impl::read_single_block(mem_addr_t blk_idx, - mem_addr_t addr, size_t length, - void *data) const { - if ((addr + length) > (blk_idx + 1) * BSIZE) { - printf( - "GPGPU-Sim PTX: ERROR * access to memory \'%s\' is unaligned : " - "addr=0x%x, length=%zu\n", - m_name.c_str(), addr, length); - printf( - "GPGPU-Sim PTX: (addr+length)=0x%lx > 0x%x=(index+1)*BSIZE, " - "index=0x%x, BSIZE=0x%x\n", - (addr + length), (blk_idx + 1) * BSIZE, blk_idx, BSIZE); - throw 1; - } - typename map_t::const_iterator i = m_data.find(blk_idx); - if (i == m_data.end()) { - for (size_t n = 0; n < length; n++) - ((unsigned char *)data)[n] = (unsigned char)0; - // printf("GPGPU-Sim PTX: WARNING reading %zu bytes from unititialized - // memory at address 0x%x in space %s\n", length, addr, m_name.c_str() ); - } else { - unsigned offset = addr & (BSIZE - 1); - unsigned nbytes = length; - i->second.read(offset, nbytes, (unsigned char *)data); - } +template void memory_space_impl::read_single_block( mem_addr_t blk_idx, mem_addr_t addr, size_t length, void *data) const +{ + if ((addr + length) > (blk_idx + 1) * BSIZE) { + printf("GPGPU-Sim PTX: ERROR * access to memory \'%s\' is unaligned : addr=0x%x, length=%zu\n", + m_name.c_str(), addr, length); + printf("GPGPU-Sim PTX: (addr+length)=0x%lx > 0x%x=(index+1)*BSIZE, index=0x%x, BSIZE=0x%x\n", + (addr+length),(blk_idx+1)*BSIZE, blk_idx, BSIZE); + throw 1; + } + typename map_t::const_iterator i = m_data.find(blk_idx); + if( i == m_data.end() ) { + for( size_t n=0; n < length; n++ ) + ((unsigned char*)data)[n] = (unsigned char) 0; + //printf("GPGPU-Sim PTX: WARNING reading %zu bytes from unititialized memory at address 0x%x in space %s\n", length, addr, m_name.c_str() ); + } else { + unsigned offset = addr & (BSIZE-1); + unsigned nbytes = length; + i->second.read(offset,nbytes,(unsigned char*)data); + } } -template -void memory_space_impl::read(mem_addr_t addr, size_t length, - void *data) const { - mem_addr_t index = addr >> m_log2_block_size; - if ((addr + length) <= (index + 1) * BSIZE) { - // fast route for intra-block access - read_single_block(index, addr, length, data); - } else { - // slow route for inter-block access - unsigned nbytes_remain = length; - unsigned dst_offset = 0; - mem_addr_t current_addr = addr; - - while (nbytes_remain > 0) { - unsigned offset = current_addr & (BSIZE - 1); - mem_addr_t page = current_addr >> m_log2_block_size; - mem_addr_t access_limit = offset + nbytes_remain; - if (access_limit > BSIZE) { - access_limit = BSIZE; +template void memory_space_impl::read( mem_addr_t addr, size_t length, void *data ) const +{ + mem_addr_t index = addr >> m_log2_block_size; + if ((addr+length) <= (index+1)*BSIZE ) { + // fast route for intra-block access + read_single_block(index, addr, length, data); + } else { + // slow route for inter-block access + unsigned nbytes_remain = length; + unsigned dst_offset = 0; + mem_addr_t current_addr = addr; + + while (nbytes_remain > 0) { + unsigned offset = current_addr & (BSIZE-1); + mem_addr_t page = current_addr >> m_log2_block_size; + mem_addr_t access_limit = offset + nbytes_remain; + if (access_limit > BSIZE) { + access_limit = BSIZE; + } + + size_t tx_bytes = access_limit - offset; + read_single_block(page, current_addr, tx_bytes, &((unsigned char*)data)[dst_offset]); + + // advance pointers + dst_offset += tx_bytes; + current_addr += tx_bytes; + nbytes_remain -= tx_bytes; } - - size_t tx_bytes = access_limit - offset; - read_single_block(page, current_addr, tx_bytes, - &((unsigned char *)data)[dst_offset]); - - // advance pointers - dst_offset += tx_bytes; - current_addr += tx_bytes; - nbytes_remain -= tx_bytes; - } - assert(nbytes_remain == 0); - } + assert(nbytes_remain == 0); + } } -template -void memory_space_impl::print(const char *format, FILE *fout) const { - typename map_t::const_iterator i_page; +template void memory_space_impl::print( const char *format, FILE *fout ) const +{ + typename map_t::const_iterator i_page; - for (i_page = m_data.begin(); i_page != m_data.end(); ++i_page) { - fprintf(fout, "%s %08x:", m_name.c_str(), i_page->first); - i_page->second.print(format, fout); - } + for ( i_page = m_data.begin(); i_page != m_data.end(); ++i_page) { + fprintf(fout, "%s %08x:", m_name.c_str(), i_page->first); + i_page->second.print(format, fout); + } } -template -void memory_space_impl::set_watch(addr_t addr, unsigned watchpoint) { - m_watchpoints[watchpoint] = addr; +template void memory_space_impl::set_watch( addr_t addr, unsigned watchpoint ) +{ + m_watchpoints[watchpoint]=addr; } template class memory_space_impl<32>; template class memory_space_impl<64>; template class memory_space_impl<8192>; -template class memory_space_impl<16 * 1024>; +template class memory_space_impl<16*1024>; -void g_print_memory_space(memory_space *mem, const char *format = "%08x", - FILE *fout = stdout) { - mem->print(format, fout); +void g_print_memory_space(memory_space *mem, const char *format = "%08x", FILE *fout = stdout) +{ + mem->print(format,fout); } #ifdef UNIT_TEST -int main(int argc, char *argv[]) { - int errors_found = 0; - memory_space *mem = new memory_space_impl<32>("test", 4); - // write address to [address] - for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 4) - mem->write(addr, 4, &addr, NULL, NULL); - - for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 4) { - unsigned tmp = 0; - mem->read(addr, 4, &tmp); - if (tmp != addr) { - errors_found = 1; - printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, addr); - } - } - - for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 1) { - unsigned char val = (addr + 128) % 256; - mem->write(addr, 1, &val, NULL, NULL); - } - - for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 1) { - unsigned tmp = 0; - mem->read(addr, 1, &tmp); - unsigned char val = (addr + 128) % 256; - if (tmp != val) { - errors_found = 1; - printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, - (unsigned)val); - } - } - - if (errors_found) { - printf("SUMMARY: ERRORS FOUND\n"); - } else { - printf("SUMMARY: UNIT TEST PASSED\n"); - } +int main(int argc, char *argv[] ) +{ + int errors_found=0; + memory_space *mem = new memory_space_impl<32>("test",4); + // write address to [address] + for( mem_addr_t addr=0; addr < 16*1024; addr+=4) + mem->write(addr,4,&addr,NULL,NULL); + + for( mem_addr_t addr=0; addr < 16*1024; addr+=4) { + unsigned tmp=0; + mem->read(addr,4,&tmp); + if( tmp != addr ) { + errors_found=1; + printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, addr ); + } + } + + for( mem_addr_t addr=0; addr < 16*1024; addr+=1) { + unsigned char val = (addr + 128) % 256; + mem->write(addr,1,&val,NULL,NULL); + } + + for( mem_addr_t addr=0; addr < 16*1024; addr+=1) { + unsigned tmp=0; + mem->read(addr,1,&tmp); + unsigned char val = (addr + 128) % 256; + if( tmp != val ) { + errors_found=1; + printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, (unsigned)val ); + } + } + + if( errors_found ) { + printf("SUMMARY: ERRORS FOUND\n"); + } else { + printf("SUMMARY: UNIT TEST PASSED\n"); + } } #endif diff --git a/src/cuda-sim/memory.h b/src/cuda-sim/memory.h index 31ad4df..ab588bc 100644 --- a/src/cuda-sim/memory.h +++ b/src/cuda-sim/memory.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -35,97 +33,101 @@ #include "../tr1_hash_map.h" #define mem_map tr1_hash_map #if tr1_hash_map_ismap == 1 -#define MEM_MAP_RESIZE(hash_size) + #define MEM_MAP_RESIZE(hash_size) #else -#define MEM_MAP_RESIZE(hash_size) (m_data.rehash(hash_size)) + #define MEM_MAP_RESIZE(hash_size) (m_data.rehash(hash_size)) #endif #include -#include -#include #include -#include +#include #include +#include +#include typedef address_type mem_addr_t; -#define MEM_BLOCK_SIZE (4 * 1024) - -template -class mem_storage { - public: - mem_storage(const mem_storage &another) { - m_data = (unsigned char *)calloc(1, BSIZE); - memcpy(m_data, another.m_data, BSIZE); - } - mem_storage() { m_data = (unsigned char *)calloc(1, BSIZE); } - ~mem_storage() { free(m_data); } - - void write(unsigned offset, size_t length, const unsigned char *data) { - assert(offset + length <= BSIZE); - memcpy(m_data + offset, data, length); - } - - void read(unsigned offset, size_t length, unsigned char *data) const { - assert(offset + length <= BSIZE); - memcpy(data, m_data + offset, length); - } - - void print(const char *format, FILE *fout) const { - unsigned int *i_data = (unsigned int *)m_data; - for (int d = 0; d < (BSIZE / sizeof(unsigned int)); d++) { - if (d % 1 == 0) { - fprintf(fout, "\n"); +#define MEM_BLOCK_SIZE (4*1024) + +template class mem_storage { +public: + mem_storage( const mem_storage &another ) + { + m_data = (unsigned char*)calloc(1,BSIZE); + memcpy(m_data,another.m_data,BSIZE); + } + mem_storage() + { + m_data = (unsigned char*)calloc(1,BSIZE); + } + ~mem_storage() + { + free(m_data); + } + + void write( unsigned offset, size_t length, const unsigned char *data ) + { + assert( offset + length <= BSIZE ); + memcpy(m_data+offset,data,length); + } + + void read( unsigned offset, size_t length, unsigned char *data ) const + { + assert( offset + length <= BSIZE ); + memcpy(data,m_data+offset,length); + } + + void print( const char *format, FILE *fout ) const + { + unsigned int *i_data = (unsigned int*)m_data; + for (int d = 0; d < (BSIZE / sizeof(unsigned int)); d++) { + if (d % 1 == 0) { + fprintf(fout, "\n"); + } + fprintf(fout, format, i_data[d]); + fprintf(fout, " "); } - fprintf(fout, format, i_data[d]); - fprintf(fout, " "); - } - fprintf(fout, "\n"); - fflush(fout); - } - - private: - unsigned m_nbytes; - unsigned char *m_data; + fprintf(fout, "\n"); + fflush(fout); + } + +private: + unsigned m_nbytes; + unsigned char *m_data; }; class ptx_thread_info; class ptx_instruction; -class memory_space { - public: - virtual ~memory_space() {} - virtual void write(mem_addr_t addr, size_t length, const void *data, - ptx_thread_info *thd, const ptx_instruction *pI) = 0; - virtual void write_only(mem_addr_t index, mem_addr_t offset, size_t length, - const void *data) = 0; - virtual void read(mem_addr_t addr, size_t length, void *data) const = 0; - virtual void print(const char *format, FILE *fout) const = 0; - virtual void set_watch(addr_t addr, unsigned watchpoint) = 0; +class memory_space +{ +public: + virtual ~memory_space() {} + virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ) = 0; + virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data ) = 0; + virtual void read( mem_addr_t addr, size_t length, void *data ) const = 0; + virtual void print( const char *format, FILE *fout ) const = 0; + virtual void set_watch( addr_t addr, unsigned watchpoint ) = 0; }; -template -class memory_space_impl : public memory_space { - public: - memory_space_impl(std::string name, unsigned hash_size); - - virtual void write(mem_addr_t addr, size_t length, const void *data, - ptx_thread_info *thd, const ptx_instruction *pI); - virtual void write_only(mem_addr_t index, mem_addr_t offset, size_t length, - const void *data); - virtual void read(mem_addr_t addr, size_t length, void *data) const; - virtual void print(const char *format, FILE *fout) const; - - virtual void set_watch(addr_t addr, unsigned watchpoint); - - private: - void read_single_block(mem_addr_t blk_idx, mem_addr_t addr, size_t length, - void *data) const; - std::string m_name; - unsigned m_log2_block_size; - typedef mem_map > map_t; - map_t m_data; - std::map m_watchpoints; +template class memory_space_impl : public memory_space { +public: + memory_space_impl( std::string name, unsigned hash_size ); + + virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ); + virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data); + virtual void read( mem_addr_t addr, size_t length, void *data ) const; + virtual void print( const char *format, FILE *fout ) const; + + virtual void set_watch( addr_t addr, unsigned watchpoint ); + +private: + void read_single_block( mem_addr_t blk_idx, mem_addr_t addr, size_t length, void *data) const; + std::string m_name; + unsigned m_log2_block_size; + typedef mem_map > map_t; + map_t m_data; + std::map m_watchpoints; }; #endif diff --git a/src/cuda-sim/opcodes.h b/src/cuda-sim/opcodes.h index 479b1ca..86d3b99 100644 --- a/src/cuda-sim/opcodes.h +++ b/src/cuda-sim/opcodes.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -31,48 +29,49 @@ #define opcodes_h_included enum opcode_t { -#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) OP, -#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) OP, +#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP, +#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP, #include "opcodes.def" - NUM_OPCODES + NUM_OPCODES #undef OP_DEF #undef OP_W_DEF }; enum special_regs { - CLOCK_REG, - HALFCLOCK_ID, - CLOCK64_REG, - CTAID_REG, - ENVREG_REG, - GRIDID_REG, - LANEID_REG, - LANEMASK_EQ_REG, - LANEMASK_LE_REG, - LANEMASK_LT_REG, - LANEMASK_GE_REG, - LANEMASK_GT_REG, - NCTAID_REG, - NTID_REG, - NSMID_REG, - NWARPID_REG, - PM_REG, - SMID_REG, - TID_REG, - WARPID_REG, - WARPSZ_REG + CLOCK_REG, + HALFCLOCK_ID, + CLOCK64_REG, + CTAID_REG, + ENVREG_REG, + GRIDID_REG, + LANEID_REG, + LANEMASK_EQ_REG, + LANEMASK_LE_REG, + LANEMASK_LT_REG, + LANEMASK_GE_REG, + LANEMASK_GT_REG, + NCTAID_REG, + NTID_REG, + NSMID_REG, + NWARPID_REG, + PM_REG, + SMID_REG, + TID_REG, + WARPID_REG, + WARPSZ_REG }; -enum wmma_type { - LOAD_A, - LOAD_B, - LOAD_C, - STORE_D, - MMA, - ROW, - COL, - M16N16K16, - M32N8K16, - M8N32K16 +enum wmma_type{ + LOAD_A, + LOAD_B, + LOAD_C, + STORE_D, + MMA, + ROW, + COL, + M16N16K16, + M32N8K16, + M8N32K16 + }; #endif diff --git a/src/cuda-sim/ptx-stats.cc b/src/cuda-sim/ptx-stats.cc index 2f16555..22517df 100644 --- a/src/cuda-sim/ptx-stats.cc +++ b/src/cuda-sim/ptx-stats.cc @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -27,266 +25,248 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "ptx_ir.h" +#include "ptx_sim.h" #include "ptx-stats.h" +#include "../option_parser.h" #include #include -#include "../../libcuda/gpgpu_context.h" -#include "../option_parser.h" #include "../tr1_hash_map.h" -#include "ptx_ir.h" -#include "ptx_sim.h" +#include "../../libcuda/gpgpu_context.h" -void ptx_stats::ptx_file_line_stats_options(option_parser_t opp) { - option_parser_register( - opp, "-enable_ptx_file_line_stats", OPT_BOOL, &enable_ptx_file_line_stats, - "Turn on PTX source line statistic profiling. (1 = On)", "1"); - option_parser_register( - opp, "-ptx_line_stats_filename", OPT_CSTR, &ptx_line_stats_filename, - "Output file for PTX source line statistics.", "gpgpu_inst_stats.txt"); +void ptx_stats::ptx_file_line_stats_options(option_parser_t opp) +{ + option_parser_register(opp, "-enable_ptx_file_line_stats", OPT_BOOL, + &enable_ptx_file_line_stats, + "Turn on PTX source line statistic profiling. (1 = On)", "1"); + option_parser_register(opp, "-ptx_line_stats_filename", OPT_CSTR, + &ptx_line_stats_filename, + "Output file for PTX source line statistics.", "gpgpu_inst_stats.txt"); } // implementations // defining a PTX source line = filename + line number -class ptx_file_line { - public: - ptx_file_line(const char *s, int l) { - if (s == NULL) - st = "NULL_NAME"; - else - st = s; - line = l; - } - - bool operator<(const ptx_file_line &other) const { - if (st == other.st) { - if (line < other.line) - return true; - else - return false; - } else { - return st < other.st; +class ptx_file_line +{ +public: + ptx_file_line(const char* s, int l) { + if( s == NULL ) + st = "NULL_NAME"; + else + st = s; + line = l; } - } - bool operator==(const ptx_file_line &other) const { - return (line == other.line) && (st == other.st); - } + bool operator<(const ptx_file_line &other) const { + if( st == other.st ) { + if( line < other.line ) + return true; + else + return false; + } else { + return st < other.st; + } + } - std::string st; - unsigned line; + bool operator==(const ptx_file_line &other) const { + return (line==other.line) && (st==other.st); + } + + std::string st; + unsigned line; }; // holds all statistics collected for a singe PTX source line -class ptx_file_line_stats { - public: - ptx_file_line_stats() - : exec_count(0), - latency(0), - dram_traffic(0), - smem_n_way_bank_conflict_total(0), - smem_warp_count(0), - gmem_n_access_total(0), - gmem_warp_count(0), - exposed_latency(0), - warp_divergence(0) {} - - unsigned long exec_count; - unsigned long long latency; - unsigned long long dram_traffic; - unsigned long long smem_n_way_bank_conflict_total; // total number of banks - // accessed by this - // instruction - unsigned long smem_warp_count; // number of warps accessing shared memory - unsigned long long gmem_n_access_total; // number of uncoalesced access in - // total from this instruction - unsigned long - gmem_warp_count; // number of warps causing these uncoalesced access - unsigned long long exposed_latency; // latency exposed as pipeline bubbles - // (attributed to this instruction) - unsigned long long - warp_divergence; // number of warp divergence occured at this instruction +class ptx_file_line_stats +{ +public: + ptx_file_line_stats() + : exec_count(0), latency(0), dram_traffic(0), + smem_n_way_bank_conflict_total(0), smem_warp_count(0), + gmem_n_access_total(0), gmem_warp_count(0), exposed_latency(0), + warp_divergence(0) + { } + + unsigned long exec_count; + unsigned long long latency; + unsigned long long dram_traffic; + unsigned long long smem_n_way_bank_conflict_total; // total number of banks accessed by this instruction + unsigned long smem_warp_count; // number of warps accessing shared memory + unsigned long long gmem_n_access_total; // number of uncoalesced access in total from this instruction + unsigned long gmem_warp_count; // number of warps causing these uncoalesced access + unsigned long long exposed_latency; // latency exposed as pipeline bubbles (attributed to this instruction) + unsigned long long warp_divergence; // number of warp divergence occured at this instruction }; #if (tr1_hash_map_ismap == 1) -typedef tr1_hash_map - ptx_file_line_stats_map_t; +typedef tr1_hash_map ptx_file_line_stats_map_t; #else -struct hash_ptx_file_line { - std::size_t operator()(const ptx_file_line &pfline) const { - std::hash hash_line; - return hash_line(pfline.line); - } +struct hash_ptx_file_line +{ + std::size_t operator()(const ptx_file_line & pfline) const { + std::hash hash_line; + return hash_line(pfline.line); + } }; -typedef tr1_hash_map - ptx_file_line_stats_map_t; +typedef tr1_hash_map ptx_file_line_stats_map_t; #endif static ptx_file_line_stats_map_t ptx_file_line_stats_tracker; // output statistics to a file -void ptx_stats::ptx_file_line_stats_write_file() { - // check if stat collection is turned on - if (enable_ptx_file_line_stats == 0) return; - - ptx_file_line_stats_map_t::iterator it; - FILE *pfile; - - pfile = fopen(ptx_line_stats_filename, "w"); - fprintf(pfile, - "kernel line : count latency dram_traffic smem_bk_conflicts " - "smem_warp gmem_access_generated gmem_warp exposed_latency " - "warp_divergence\n"); - for (it = ptx_file_line_stats_tracker.begin(); - it != ptx_file_line_stats_tracker.end(); it++) { - fprintf(pfile, "%s %i : ", it->first.st.c_str(), it->first.line); - fprintf(pfile, "%lu ", it->second.exec_count); - fprintf(pfile, "%llu ", it->second.latency); - fprintf(pfile, "%llu ", it->second.dram_traffic); - fprintf(pfile, "%llu ", it->second.smem_n_way_bank_conflict_total); - fprintf(pfile, "%lu ", it->second.smem_warp_count); - fprintf(pfile, "%llu ", it->second.gmem_n_access_total); - fprintf(pfile, "%lu ", it->second.gmem_warp_count); - fprintf(pfile, "%llu ", it->second.exposed_latency); - fprintf(pfile, "%llu ", it->second.warp_divergence); - fprintf(pfile, "\n"); - } - fflush(pfile); - fclose(pfile); +void ptx_stats::ptx_file_line_stats_write_file() +{ + // check if stat collection is turned on + if (enable_ptx_file_line_stats == 0) return; + + ptx_file_line_stats_map_t::iterator it; + FILE * pfile; + + pfile = fopen(ptx_line_stats_filename, "w"); + fprintf(pfile,"kernel line : count latency dram_traffic smem_bk_conflicts smem_warp gmem_access_generated gmem_warp exposed_latency warp_divergence\n"); + for( it=ptx_file_line_stats_tracker.begin(); it != ptx_file_line_stats_tracker.end(); it++ ) { + fprintf(pfile, "%s %i : ", it->first.st.c_str(), it->first.line); + fprintf(pfile, "%lu ", it->second.exec_count); + fprintf(pfile, "%llu ", it->second.latency); + fprintf(pfile, "%llu ", it->second.dram_traffic); + fprintf(pfile, "%llu ", it->second.smem_n_way_bank_conflict_total); + fprintf(pfile, "%lu ", it->second.smem_warp_count); + fprintf(pfile, "%llu ", it->second.gmem_n_access_total); + fprintf(pfile, "%lu ", it->second.gmem_warp_count); + fprintf(pfile, "%llu ", it->second.exposed_latency); + fprintf(pfile, "%llu ", it->second.warp_divergence); + fprintf(pfile, "\n"); + } + fflush(pfile); + fclose(pfile); } // attribute one more execution count to this ptx instruction // counting the number of threads (not warps) executing this instruction -void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn) { - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), - pInsn->source_line())] - .exec_count += 1; +void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn) +{ + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].exec_count += 1; } // attribute pipeline latency to this ptx instruction (specified by the pc) -// pipeline latency is the number of cycles a warp with this instruction spent -// in the pipeline -void ptx_stats::ptx_file_line_stats_add_latency(unsigned pc, unsigned latency) { - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), - pInsn->source_line())] - .latency += latency; +// pipeline latency is the number of cycles a warp with this instruction spent in the pipeline +void ptx_stats::ptx_file_line_stats_add_latency(unsigned pc, unsigned latency) +{ + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].latency += latency; } // attribute dram traffic to this ptx instruction (specified by the pc) -// dram traffic is counted in number of requests -void ptx_stats::ptx_file_line_stats_add_dram_traffic(unsigned pc, - unsigned dram_traffic) { - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), - pInsn->source_line())] - .dram_traffic += dram_traffic; +// dram traffic is counted in number of requests +void ptx_stats::ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic) +{ + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].dram_traffic += dram_traffic; } // attribute the number of shared memory access cycles to a ptx instruction -// counts both the number of warps doing shared memory access and the number of -// cycles involved -void ptx_stats::ptx_file_line_stats_add_smem_bank_conflict( - unsigned pc, unsigned n_way_bkconflict) { - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( - pInsn->source_file(), pInsn->source_line())]; - line_stats.smem_n_way_bank_conflict_total += n_way_bkconflict; - line_stats.smem_warp_count += 1; +// counts both the number of warps doing shared memory access and the number of cycles involved +void ptx_stats::ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, unsigned n_way_bkconflict) +{ + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; + line_stats.smem_n_way_bank_conflict_total += n_way_bkconflict; + line_stats.smem_warp_count += 1; } -// attribute a non-coalesced mem access to a ptx instruction -// counts both the number of warps causing this and the number of memory -// requests generated -void ptx_stats::ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, - unsigned n_access) { - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( - pInsn->source_file(), pInsn->source_line())]; - line_stats.gmem_n_access_total += n_access; - line_stats.gmem_warp_count += 1; +// attribute a non-coalesced mem access to a ptx instruction +// counts both the number of warps causing this and the number of memory requests generated +void ptx_stats::ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access) +{ + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; + line_stats.gmem_n_access_total += n_access; + line_stats.gmem_warp_count += 1; } -// a class that tracks the inflight memory instructions of a shader core -// and attributes exposed latency to those instructions when signaled to do so -class ptx_inflight_memory_insn_tracker { - public: - typedef std::map insn_count_map; +// a class that tracks the inflight memory instructions of a shader core +// and attributes exposed latency to those instructions when signaled to do so +class ptx_inflight_memory_insn_tracker +{ +public: + typedef std::map insn_count_map; - void add_count(const ptx_instruction *pInsn, int count = 1) { - ptx_inflight_memory_insns[pInsn] += count; - } + void add_count(const ptx_instruction * pInsn, int count = 1) + { + ptx_inflight_memory_insns[pInsn] += count; + } - void sub_count(const ptx_instruction *pInsn, int count = 1) { - insn_count_map::iterator i_insncount; - i_insncount = ptx_inflight_memory_insns.find(pInsn); + void sub_count(const ptx_instruction * pInsn, int count = 1) + { + insn_count_map::iterator i_insncount; + i_insncount = ptx_inflight_memory_insns.find(pInsn); - assert(i_insncount != ptx_inflight_memory_insns.end()); + assert(i_insncount != ptx_inflight_memory_insns.end()); - i_insncount->second -= count; + i_insncount->second -= count; - if (i_insncount->second <= 0) { - ptx_inflight_memory_insns.erase(i_insncount); + if (i_insncount->second <= 0) { + ptx_inflight_memory_insns.erase(i_insncount); + } } - } - - void attribute_exposed_latency(int count = 1) { - insn_count_map &exlat_insnmap = ptx_inflight_memory_insns; - insn_count_map::const_iterator i_exlatinsn; - - i_exlatinsn = exlat_insnmap.begin(); - for (; i_exlatinsn != exlat_insnmap.end(); ++i_exlatinsn) { - const ptx_instruction *pInsn = i_exlatinsn->first; - ptx_file_line_stats &line_stats = - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), - pInsn->source_line())]; - line_stats.exposed_latency += count; + + void attribute_exposed_latency(int count = 1) + { + insn_count_map &exlat_insnmap = ptx_inflight_memory_insns; + insn_count_map::const_iterator i_exlatinsn; + + i_exlatinsn = exlat_insnmap.begin(); + for (; i_exlatinsn != exlat_insnmap.end(); ++i_exlatinsn) { + const ptx_instruction *pInsn = i_exlatinsn->first; + ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; + line_stats.exposed_latency += count; + } } - } - insn_count_map ptx_inflight_memory_insns; + insn_count_map ptx_inflight_memory_insns; }; static ptx_inflight_memory_insn_tracker *inflight_mem_tracker = NULL; -void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores) { - inflight_mem_tracker = new ptx_inflight_memory_insn_tracker[n_shader_cores]; +void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores) +{ + inflight_mem_tracker = new ptx_inflight_memory_insn_tracker[n_shader_cores]; } // add an inflight memory instruction -void ptx_stats::ptx_file_line_stats_add_inflight_memory_insn(int sc_id, - unsigned pc) { - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); +void ptx_stats::ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc) +{ + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - inflight_mem_tracker[sc_id].add_count(pInsn); + inflight_mem_tracker[sc_id].add_count(pInsn); } // remove an inflight memory instruction -void ptx_stats::ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, - unsigned pc) { - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); +void ptx_stats::ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc) +{ + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - inflight_mem_tracker[sc_id].sub_count(pInsn); + inflight_mem_tracker[sc_id].sub_count(pInsn); } -// attribute an empty cycle in the pipeline (exposed latency) to the ptx memory -// instructions in flight -void ptx_file_line_stats_commit_exposed_latency(int sc_id, - int exposed_latency) { - assert(exposed_latency > 0); - inflight_mem_tracker[sc_id].attribute_exposed_latency(exposed_latency); +// attribute an empty cycle in the pipeline (exposed latency) to the ptx memory instructions in flight +void ptx_file_line_stats_commit_exposed_latency(int sc_id, int exposed_latency) +{ + assert(exposed_latency > 0); + inflight_mem_tracker[sc_id].attribute_exposed_latency(exposed_latency); } // attribute the number of warp divergence to a ptx instruction -void ptx_stats::ptx_file_line_stats_add_warp_divergence( - unsigned pc, unsigned n_way_divergence) { - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( - pInsn->source_file(), pInsn->source_line())]; - line_stats.warp_divergence += n_way_divergence; +void ptx_stats::ptx_file_line_stats_add_warp_divergence(unsigned pc, unsigned n_way_divergence) +{ + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; + line_stats.warp_divergence += n_way_divergence; } + diff --git a/src/cuda-sim/ptx-stats.h b/src/cuda-sim/ptx-stats.h index 7745eba..246b4ce 100644 --- a/src/cuda-sim/ptx-stats.h +++ b/src/cuda-sim/ptx-stats.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -27,14 +25,15 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#pragma once +#pragma once #include "../option_parser.h" + #ifdef __cplusplus // stat collection interface to cuda-sim class ptx_instruction; -void ptx_file_line_stats_add_exec_count(const ptx_instruction* pInsn); +void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn); #endif // stat collection interface to gpgpu-sim @@ -42,29 +41,28 @@ void ptx_file_line_stats_add_exec_count(const ptx_instruction* pInsn); void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores); void ptx_file_line_stats_commit_exposed_latency(int sc_id, int exposed_latency); + class gpgpu_context; class ptx_stats { - public: - ptx_stats(gpgpu_context* ctx) { - ptx_line_stats_filename = NULL; - gpgpu_ctx = ctx; - } - char* ptx_line_stats_filename; - bool enable_ptx_file_line_stats; - gpgpu_context* gpgpu_ctx; - // set options - void ptx_file_line_stats_options(option_parser_t opp); + public: + ptx_stats(gpgpu_context* ctx) { + ptx_line_stats_filename = NULL; + gpgpu_ctx = ctx; + } + char * ptx_line_stats_filename; + bool enable_ptx_file_line_stats; + gpgpu_context* gpgpu_ctx; + // set options + void ptx_file_line_stats_options(option_parser_t opp); - // output stats to a file - void ptx_file_line_stats_write_file(); - // stat collection interface to gpgpu-sim - void ptx_file_line_stats_add_latency(unsigned pc, unsigned latency); - void ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic); - void ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, - unsigned n_way_bkconflict); - void ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access); - void ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc); - void ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc); - void ptx_file_line_stats_add_warp_divergence(unsigned pc, - unsigned n_way_divergence); + // output stats to a file + void ptx_file_line_stats_write_file(); + // stat collection interface to gpgpu-sim + void ptx_file_line_stats_add_latency(unsigned pc, unsigned latency); + void ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic); + void ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, unsigned n_way_bkconflict); + void ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access); + void ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc); + void ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc); + void ptx_file_line_stats_add_warp_divergence(unsigned pc, unsigned n_way_divergence); }; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index 55afb51..d8943d2 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -1,5 +1,5 @@ // Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda, Wilson W.L. Fung, -// George L. Yuan +// George L. Yuan // The University of British Columbia // All rights reserved. // @@ -8,16 +8,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -28,1244 +26,1152 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#include "ptx_ir.h" #include "ptx_parser.h" -typedef void *yyscan_t; -#include +#include "ptx_ir.h" +typedef void * yyscan_t; +#include "ptx.tab.h" +#include "opcodes.h" #include #include -#include #include +#include +#include #include "assert.h" -#include "opcodes.h" -#include "ptx.tab.h" -#include "../../libcuda/gpgpu_context.h" #include "cuda-sim.h" +#include "../../libcuda/gpgpu_context.h" #define STR_SIZE 1024 -const ptx_instruction *gpgpu_context::pc_to_instruction(unsigned pc) { - if (pc < s_g_pc_to_insn.size()) - return s_g_pc_to_insn[pc]; - else - return NULL; +const ptx_instruction* gpgpu_context::pc_to_instruction(unsigned pc) +{ + if( pc < s_g_pc_to_insn.size() ) + return s_g_pc_to_insn[pc]; + else + return NULL; } -unsigned symbol::get_uid() { - unsigned result = (gpgpu_ctx->symbol_sm_next_uid)++; - return result; +unsigned symbol::get_uid() +{ + unsigned result = (gpgpu_ctx->symbol_sm_next_uid)++; + return result; } -void symbol::add_initializer(const std::list &init) { - m_initializer = init; +void symbol::add_initializer( const std::list &init ) +{ + m_initializer = init; } -void symbol::print_info(FILE *fp) const { - fprintf(fp, "uid:%u, decl:%s, type:%p, ", m_uid, m_decl_location.c_str(), - m_type); - if (m_address_valid) fprintf(fp, "
, "); - if (m_is_label) fprintf(fp, " is_label "); - if (m_is_shared) fprintf(fp, " is_shared "); - if (m_is_const) fprintf(fp, " is_const "); - if (m_is_global) fprintf(fp, " is_global "); - if (m_is_local) fprintf(fp, " is_local "); - if (m_is_tex) fprintf(fp, " is_tex "); - if (m_is_func_addr) fprintf(fp, " is_func_addr "); - if (m_function) fprintf(fp, " %p ", m_function); +void symbol::print_info(FILE *fp) const +{ + fprintf(fp,"uid:%u, decl:%s, type:%p, ", m_uid, m_decl_location.c_str(), m_type ); + if( m_address_valid ) + fprintf(fp,"
, "); + if( m_is_label ) + fprintf(fp," is_label "); + if( m_is_shared ) + fprintf(fp," is_shared "); + if( m_is_const ) + fprintf(fp," is_const "); + if( m_is_global ) + fprintf(fp," is_global "); + if( m_is_local ) + fprintf(fp," is_local "); + if( m_is_tex ) + fprintf(fp," is_tex "); + if( m_is_func_addr ) + fprintf(fp," is_func_addr "); + if( m_function ) + fprintf(fp," %p ", m_function ); } -symbol_table::symbol_table() { assert(0); } +symbol_table::symbol_table() +{ + assert(0); +} -symbol_table::symbol_table(const char *scope_name, unsigned entry_point, - symbol_table *parent, gpgpu_context *ctx) { - gpgpu_ctx = ctx; - m_scope_name = std::string(scope_name); - m_reg_allocator = 0; - m_shared_next = 0; - m_const_next = 0; - m_global_next = 0x100; - m_local_next = 0; - m_tex_next = 0; - - // Jin: handle instruction group for cdp - m_inst_group_id = 0; +symbol_table::symbol_table( const char *scope_name, unsigned entry_point, symbol_table *parent, gpgpu_context* ctx ) +{ + gpgpu_ctx = ctx; + m_scope_name = std::string(scope_name); + m_reg_allocator=0; + m_shared_next = 0; + m_const_next = 0; + m_global_next = 0x100; + m_local_next = 0; + m_tex_next = 0; + + //Jin: handle instruction group for cdp + m_inst_group_id = 0; + + m_parent = parent; + if ( m_parent ) { + m_shared_next = m_parent->m_shared_next; + m_global_next = m_parent->m_global_next; + } +} - m_parent = parent; - if (m_parent) { - m_shared_next = m_parent->m_shared_next; - m_global_next = m_parent->m_global_next; - } +void symbol_table::set_name( const char *name ) +{ + m_scope_name = std::string(name); } -void symbol_table::set_name(const char *name) { - m_scope_name = std::string(name); +const ptx_version &symbol_table::get_ptx_version() const +{ + if( m_parent == NULL ) return m_ptx_version; + else return m_parent->get_ptx_version(); } -const ptx_version &symbol_table::get_ptx_version() const { - if (m_parent == NULL) - return m_ptx_version; - else - return m_parent->get_ptx_version(); +unsigned symbol_table::get_sm_target() const +{ + if( m_parent == NULL ) + return m_ptx_version.target(); + else return m_parent->get_sm_target(); } -unsigned symbol_table::get_sm_target() const { - if (m_parent == NULL) - return m_ptx_version.target(); - else - return m_parent->get_sm_target(); +void symbol_table::set_ptx_version( float ver, unsigned ext ) +{ + m_ptx_version = ptx_version(ver,ext); } -void symbol_table::set_ptx_version(float ver, unsigned ext) { - m_ptx_version = ptx_version(ver, ext); +void symbol_table::set_sm_target( const char *target, const char *ext, const char *ext2 ) +{ + m_ptx_version.set_target(target,ext,ext2); } - -void symbol_table::set_sm_target(const char *target, const char *ext, - const char *ext2) { - m_ptx_version.set_target(target, ext, ext2); -} - -symbol *symbol_table::lookup(const char *identifier) { - std::string key(identifier); - std::map::iterator i = m_symbols.find(key); - if (i != m_symbols.end()) { - return i->second; - } - if (m_parent) { - return m_parent->lookup(identifier); - } - return NULL; -} - -symbol *symbol_table::add_variable(const char *identifier, - const type_info *type, unsigned size, - const char *filename, unsigned line) { - char buf[1024]; - std::string key(identifier); - assert(m_symbols.find(key) == m_symbols.end()); - snprintf(buf, 1024, "%s:%u", filename, line); - symbol *s = new symbol(identifier, type, buf, size, gpgpu_ctx); - m_symbols[key] = s; - - if (type != NULL && type->get_key().is_global()) { - m_globals.push_back(s); - } - if (type != NULL && type->get_key().is_const()) { - m_consts.push_back(s); - } - - return s; -} - -void symbol_table::add_function(function_info *func, const char *filename, - unsigned linenumber) { - std::map::iterator i = - m_symbols.find(func->get_name()); - if (i != m_symbols.end()) return; - char buf[1024]; - snprintf(buf, 1024, "%s:%u", filename, linenumber); - type_info *type = add_type(func); - symbol *s = new symbol(func->get_name().c_str(), type, buf, 0, gpgpu_ctx); - s->set_function(func); - m_symbols[func->get_name()] = s; -} - -// Jin: handle instruction group for cdp -symbol_table *symbol_table::start_inst_group() { - char inst_group_name[4096]; - snprintf(inst_group_name, 4096, "%s_inst_group_%u", m_scope_name.c_str(), - m_inst_group_id); - - // previous added - assert(m_inst_group_symtab.find(std::string(inst_group_name)) == - m_inst_group_symtab.end()); - symbol_table *sym_table = - new symbol_table(inst_group_name, 3 /*inst group*/, this, gpgpu_ctx); - - sym_table->m_global_next = m_global_next; - sym_table->m_shared_next = m_shared_next; - sym_table->m_local_next = m_local_next; - sym_table->m_reg_allocator = m_reg_allocator; - sym_table->m_tex_next = m_tex_next; - sym_table->m_const_next = m_const_next; - - m_inst_group_symtab[std::string(inst_group_name)] = sym_table; - - return sym_table; -} - -symbol_table *symbol_table::end_inst_group() { - symbol_table *sym_table = m_parent; - - sym_table->m_global_next = m_global_next; - sym_table->m_shared_next = m_shared_next; - sym_table->m_local_next = m_local_next; - sym_table->m_reg_allocator = m_reg_allocator; - sym_table->m_tex_next = m_tex_next; - sym_table->m_const_next = m_const_next; - sym_table->m_inst_group_id++; - - return sym_table; -} - -void register_ptx_function(const char *name, - function_info *impl); // either libcuda or libopencl - -bool symbol_table::add_function_decl(const char *name, int entry_point, - function_info **func_info, - symbol_table **sym_table) { - std::string key = std::string(name); - bool prior_decl = false; - if (m_function_info_lookup.find(key) != m_function_info_lookup.end()) { - *func_info = m_function_info_lookup[key]; - prior_decl = true; - } else { - *func_info = new function_info(entry_point, gpgpu_ctx); - (*func_info)->set_name(name); - (*func_info)->set_maxnt_id(0); - m_function_info_lookup[key] = *func_info; - } - - if (m_function_symtab_lookup.find(key) != m_function_symtab_lookup.end()) { - assert(prior_decl); - *sym_table = m_function_symtab_lookup[key]; - } else { - assert(!prior_decl); - *sym_table = new symbol_table("", entry_point, this, gpgpu_ctx); - - // Initial setup code to support a register represented as "_". - // This register is used when an instruction operand is - // not read or written. However, the parser must recognize it - // as a legitimate register but we do not want to pass - // it to the micro-architectural register to the performance simulator. - // For this purpose we add a symbol to the symbol table but - // mark it as a non_arch_reg so it does not effect the performance sim. - type_info_key null_key(reg_space, 0, 0, 0, 0, 0); - null_key.set_is_non_arch_reg(); - // First param is null - which is bad. - // However, the first parameter is actually unread in the constructor... - // TODO - remove the symbol_table* from type_info - type_info *null_type_info = new type_info(NULL, null_key); - symbol *null_reg = - (*sym_table)->add_variable("_", null_type_info, 0, "", 0); - null_reg->set_regno(0, 0); - - (*sym_table)->set_name(name); - (*func_info)->set_symtab(*sym_table); - m_function_symtab_lookup[key] = *sym_table; - assert((*func_info)->get_symtab() == *sym_table); - register_ptx_function(name, *func_info); - } - return prior_decl; -} - -function_info *symbol_table::lookup_function(std::string name) { - std::string key = std::string(name); - std::map::iterator it = - m_function_info_lookup.find(key); - assert(it != m_function_info_lookup.end()); - return it->second; -} - -type_info *symbol_table::add_type(memory_space_t space_spec, - int scalar_type_spec, int vector_spec, - int alignment_spec, int extern_spec) { - if (space_spec == param_space_unclassified) space_spec = param_space_local; - type_info_key t(space_spec, scalar_type_spec, vector_spec, alignment_spec, - extern_spec, 0); - type_info *pt; - pt = new type_info(this, t); - return pt; -} - -type_info *symbol_table::add_type(function_info *func) { - type_info_key t; - type_info *pt; - t.set_is_func(); - pt = new type_info(this, t); - return pt; -} - -type_info *symbol_table::get_array_type(type_info *base_type, - unsigned array_dim) { - type_info_key t = base_type->get_key(); - t.set_array_dim(array_dim); - type_info *pt = new type_info(this, t); - // Where else is m_types being used? As of now, I dont find any use of it and - // causing seg fault. So disabling m_types. - // TODO: find where m_types can be used in future and solve the seg fault. - // pt = m_types[t] = new type_info(this,t); - return pt; -} - -void symbol_table::set_label_address(const symbol *label, unsigned addr) { - std::map::iterator i = m_symbols.find(label->name()); - assert(i != m_symbols.end()); - symbol *s = i->second; - s->set_label_address(addr); -} - -void symbol_table::dump() { - printf("\n\n"); - printf("Symbol table for \"%s\":\n", m_scope_name.c_str()); - std::map::iterator i; - for (i = m_symbols.begin(); i != m_symbols.end(); i++) { - printf("%30s : ", i->first.c_str()); - if (i->second) - i->second->print_info(stdout); - else - printf(" "); - printf("\n"); - } - printf("\n"); + +symbol *symbol_table::lookup( const char *identifier ) +{ + std::string key(identifier); + std::map::iterator i = m_symbols.find(key); + if ( i != m_symbols.end() ) { + return i->second; + } + if ( m_parent ) { + return m_parent->lookup(identifier); + } + return NULL; } -unsigned operand_info::get_uid() { - unsigned result = (gpgpu_ctx->operand_info_sm_next_uid)++; - return result; +symbol *symbol_table::add_variable( const char *identifier, const type_info *type, unsigned size, const char *filename, unsigned line ) +{ + char buf[1024]; + std::string key(identifier); + assert( m_symbols.find(key) == m_symbols.end() ); + snprintf(buf,1024,"%s:%u",filename,line); + symbol *s = new symbol(identifier,type,buf,size,gpgpu_ctx); + m_symbols[ key ] = s; + + if ( type != NULL && type->get_key().is_global() ) { + m_globals.push_back(s); + } + if ( type != NULL && type->get_key().is_const() ) { + m_consts.push_back(s); + } + + return s; } -std::list::iterator -function_info::find_next_real_instruction( - std::list::iterator i) { - while ((i != m_instructions.end()) && (*i)->is_label()) i++; - return i; +void symbol_table::add_function( function_info *func, const char *filename, unsigned linenumber ) +{ + std::map::iterator i = m_symbols.find( func->get_name() ); + if( i != m_symbols.end() ) + return; + char buf[1024]; + snprintf(buf,1024,"%s:%u",filename,linenumber); + type_info *type = add_type( func ); + symbol *s = new symbol(func->get_name().c_str(),type,buf,0,gpgpu_ctx); + s->set_function(func); + m_symbols[ func->get_name() ] = s; } -void function_info::create_basic_blocks() { - std::list leaders; - std::list::iterator i, l; +//Jin: handle instruction group for cdp +symbol_table* symbol_table::start_inst_group() { + char inst_group_name[4096]; + snprintf(inst_group_name, 4096, "%s_inst_group_%u", m_scope_name.c_str(), m_inst_group_id); + + //previous added + assert(m_inst_group_symtab.find(std::string(inst_group_name)) == m_inst_group_symtab.end()); + symbol_table *sym_table = new symbol_table(inst_group_name, 3/*inst group*/, this, gpgpu_ctx ); + + sym_table->m_global_next = m_global_next; + sym_table->m_shared_next = m_shared_next; + sym_table->m_local_next = m_local_next; + sym_table->m_reg_allocator = m_reg_allocator; + sym_table->m_tex_next = m_tex_next; + sym_table->m_const_next = m_const_next; + + m_inst_group_symtab[std::string(inst_group_name)] = sym_table; + + return sym_table; +} - // first instruction is a leader - i = m_instructions.begin(); - leaders.push_back(*i); - i++; - while (i != m_instructions.end()) { - ptx_instruction *pI = *i; - if (pI->is_label()) { - leaders.push_back(pI); - i = find_next_real_instruction(++i); - } else { - switch (pI->get_opcode()) { - case BRA_OP: - case RET_OP: - case EXIT_OP: - case RETP_OP: - case BREAK_OP: - i++; - if (i != m_instructions.end()) leaders.push_back(*i); - i = find_next_real_instruction(i); - break; - case CALL_OP: - case CALLP_OP: - if (pI->has_pred()) { - printf("GPGPU-Sim PTX: Warning found predicated call\n"); +symbol_table * symbol_table::end_inst_group() { + symbol_table * sym_table = m_parent; + + sym_table->m_global_next = m_global_next; + sym_table->m_shared_next = m_shared_next; + sym_table->m_local_next = m_local_next; + sym_table->m_reg_allocator = m_reg_allocator; + sym_table->m_tex_next = m_tex_next; + sym_table->m_const_next = m_const_next; + sym_table->m_inst_group_id++; + + return sym_table; +} + +void register_ptx_function( const char *name, function_info *impl ); // either libcuda or libopencl + +bool symbol_table::add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **sym_table ) +{ + std::string key = std::string(name); + bool prior_decl = false; + if( m_function_info_lookup.find(key) != m_function_info_lookup.end() ) { + *func_info = m_function_info_lookup[key]; + prior_decl = true; + } else { + *func_info = new function_info(entry_point, gpgpu_ctx); + (*func_info)->set_name(name); + (*func_info)->set_maxnt_id(0); + m_function_info_lookup[key] = *func_info; + } + + if( m_function_symtab_lookup.find(key) != m_function_symtab_lookup.end() ) { + assert( prior_decl ); + *sym_table = m_function_symtab_lookup[key]; + } else { + assert( !prior_decl ); + *sym_table = new symbol_table( "", entry_point, this, gpgpu_ctx ); + + // Initial setup code to support a register represented as "_". + // This register is used when an instruction operand is + // not read or written. However, the parser must recognize it + // as a legitimate register but we do not want to pass + // it to the micro-architectural register to the performance simulator. + // For this purpose we add a symbol to the symbol table but + // mark it as a non_arch_reg so it does not effect the performance sim. + type_info_key null_key( reg_space, 0, 0, 0, 0, 0 ); + null_key.set_is_non_arch_reg(); + // First param is null - which is bad. + // However, the first parameter is actually unread in the constructor... + // TODO - remove the symbol_table* from type_info + type_info* null_type_info = new type_info( NULL, null_key ); + symbol *null_reg = (*sym_table)->add_variable( "_", null_type_info, 0, "", 0 ); + null_reg->set_regno(0, 0); + + (*sym_table)->set_name(name); + (*func_info)->set_symtab(*sym_table); + m_function_symtab_lookup[key] = *sym_table; + assert( (*func_info)->get_symtab() == *sym_table ); + register_ptx_function(name,*func_info); + } + return prior_decl; +} + +function_info *symbol_table::lookup_function( std::string name ) +{ + std::string key = std::string(name); + std::map::iterator it = m_function_info_lookup.find(key); + assert ( it != m_function_info_lookup.end() ); + return it->second; +} + +type_info *symbol_table::add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ) +{ + if( space_spec == param_space_unclassified ) + space_spec = param_space_local; + type_info_key t(space_spec,scalar_type_spec,vector_spec,alignment_spec,extern_spec,0); + type_info *pt; + pt = new type_info(this,t); + return pt; +} + +type_info *symbol_table::add_type( function_info *func ) +{ + type_info_key t; + type_info *pt; + t.set_is_func(); + pt = new type_info(this,t); + return pt; +} + +type_info *symbol_table::get_array_type( type_info *base_type, unsigned array_dim ) +{ + type_info_key t = base_type->get_key(); + t.set_array_dim(array_dim); + type_info *pt = new type_info(this,t); + //Where else is m_types being used? As of now, I dont find any use of it and causing seg fault. So disabling m_types. + //TODO: find where m_types can be used in future and solve the seg fault. + //pt = m_types[t] = new type_info(this,t); + return pt; +} + +void symbol_table::set_label_address( const symbol *label, unsigned addr ) +{ + std::map::iterator i=m_symbols.find(label->name()); + assert( i != m_symbols.end() ); + symbol *s = i->second; + s->set_label_address(addr); +} + +void symbol_table::dump() +{ + printf("\n\n"); + printf("Symbol table for \"%s\":\n", m_scope_name.c_str() ); + std::map::iterator i; + for( i=m_symbols.begin(); i!=m_symbols.end(); i++ ) { + printf("%30s : ", i->first.c_str() ); + if( i->second ) + i->second->print_info(stdout); + else + printf(" "); + printf("\n"); + } + printf("\n"); +} + +unsigned operand_info::get_uid() +{ + unsigned result = (gpgpu_ctx->operand_info_sm_next_uid)++; + return result; +} + +std::list::iterator function_info::find_next_real_instruction( std::list::iterator i) +{ + while( (i != m_instructions.end()) && (*i)->is_label() ) + i++; + return i; +} + +void function_info::create_basic_blocks() +{ + std::list leaders; + std::list::iterator i, l; + + // first instruction is a leader + i=m_instructions.begin(); + leaders.push_back(*i); + i++; + while( i!=m_instructions.end() ) { + ptx_instruction *pI = *i; + if( pI->is_label() ) { + leaders.push_back(pI); + i = find_next_real_instruction(++i); + } else { + switch( pI->get_opcode() ) { + case BRA_OP: case RET_OP: case EXIT_OP: case RETP_OP: case BREAK_OP: i++; - if (i != m_instructions.end()) leaders.push_back(*i); + if( i != m_instructions.end() ) + leaders.push_back(*i); i = find_next_real_instruction(i); - } else + break; + case CALL_OP: case CALLP_OP: + if( pI->has_pred() ) { + printf("GPGPU-Sim PTX: Warning found predicated call\n"); + i++; + if( i != m_instructions.end() ) + leaders.push_back(*i); + i = find_next_real_instruction(i); + } else i++; + break; + default: i++; - break; - default: - i++; - } - } - } - - if (leaders.empty()) { - printf("GPGPU-Sim PTX: Function \'%s\' has no basic blocks\n", - m_name.c_str()); - return; - } - - unsigned bb_id = 0; - l = leaders.begin(); - i = m_instructions.begin(); - m_basic_blocks.push_back( - new basic_block_t(bb_id++, *find_next_real_instruction(i), NULL, 1, 0)); - ptx_instruction *last_real_inst = *(l++); - - for (; i != m_instructions.end(); i++) { - ptx_instruction *pI = *i; - if (l != leaders.end() && *i == *l) { - // found start of next basic block - m_basic_blocks.back()->ptx_end = last_real_inst; - if (find_next_real_instruction(i) != - m_instructions.end()) { // if not bogus trailing label - m_basic_blocks.push_back(new basic_block_t( - bb_id++, *find_next_real_instruction(i), NULL, 0, 0)); - last_real_inst = *find_next_real_instruction(i); + } + } + } + + if( leaders.empty() ) { + printf("GPGPU-Sim PTX: Function \'%s\' has no basic blocks\n", m_name.c_str()); + return; + } + + unsigned bb_id = 0; + l=leaders.begin(); + i=m_instructions.begin(); + m_basic_blocks.push_back( new basic_block_t(bb_id++,*find_next_real_instruction(i),NULL,1,0) ); + ptx_instruction *last_real_inst=*(l++); + + for( ; i!=m_instructions.end(); i++ ) { + ptx_instruction *pI = *i; + if( l != leaders.end() && *i == *l ) { + // found start of next basic block + m_basic_blocks.back()->ptx_end = last_real_inst; + if( find_next_real_instruction(i) != m_instructions.end() ) { // if not bogus trailing label + m_basic_blocks.push_back( new basic_block_t(bb_id++,*find_next_real_instruction(i),NULL,0,0) ); + last_real_inst = *find_next_real_instruction(i); + } + // start search for next leader + l++; } - // start search for next leader - l++; - } - pI->assign_bb(m_basic_blocks.back()); - if (!pI->is_label()) last_real_inst = pI; - } - m_basic_blocks.back()->ptx_end = last_real_inst; - m_basic_blocks.push_back( - /*exit basic block*/ new basic_block_t(bb_id, NULL, NULL, 0, 1)); -} - -void function_info::print_basic_blocks() { - printf("Printing basic blocks for function \'%s\':\n", m_name.c_str()); - std::list::iterator ptx_itr; - unsigned last_bb = 0; - for (ptx_itr = m_instructions.begin(); ptx_itr != m_instructions.end(); - ptx_itr++) { - if ((*ptx_itr)->get_bb()) { - if ((*ptx_itr)->get_bb()->bb_id != last_bb) { - printf("\n"); - last_bb = (*ptx_itr)->get_bb()->bb_id; + pI->assign_bb( m_basic_blocks.back() ); + if( !pI->is_label() ) last_real_inst = pI; + } + m_basic_blocks.back()->ptx_end = last_real_inst; + m_basic_blocks.push_back( /*exit basic block*/ new basic_block_t(bb_id,NULL,NULL,0,1) ); +} + +void function_info::print_basic_blocks() +{ + printf("Printing basic blocks for function \'%s\':\n", m_name.c_str() ); + std::list::iterator ptx_itr; + unsigned last_bb=0; + for (ptx_itr = m_instructions.begin();ptx_itr != m_instructions.end(); ptx_itr++) { + if( (*ptx_itr)->get_bb() ) { + if( (*ptx_itr)->get_bb()->bb_id != last_bb ) { + printf("\n"); + last_bb = (*ptx_itr)->get_bb()->bb_id; + } + printf("bb_%02u\t: ", (*ptx_itr)->get_bb()->bb_id); + (*ptx_itr)->print_insn(); + printf("\n"); } - printf("bb_%02u\t: ", (*ptx_itr)->get_bb()->bb_id); - (*ptx_itr)->print_insn(); + } + printf("\nSummary of basic blocks for \'%s\':\n", m_name.c_str() ); + std::vector::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { + printf("bb_%02u\t:", (*bb_itr)->bb_id); + if ((*bb_itr)->ptx_begin) + printf(" first: %s\t", ((*bb_itr)->ptx_begin)->get_opcode_cstr()); + else printf(" first: NULL\t"); + if ((*bb_itr)->ptx_end) { + printf(" last: %s\t", ((*bb_itr)->ptx_end)->get_opcode_cstr()); + } else printf(" last: NULL\t"); printf("\n"); - } - } - printf("\nSummary of basic blocks for \'%s\':\n", m_name.c_str()); - std::vector::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); - bb_itr++) { - printf("bb_%02u\t:", (*bb_itr)->bb_id); - if ((*bb_itr)->ptx_begin) - printf(" first: %s\t", ((*bb_itr)->ptx_begin)->get_opcode_cstr()); - else - printf(" first: NULL\t"); - if ((*bb_itr)->ptx_end) { - printf(" last: %s\t", ((*bb_itr)->ptx_end)->get_opcode_cstr()); - } else - printf(" last: NULL\t"); - printf("\n"); - } - printf("\n"); -} - -void function_info::print_basic_block_links() { - printf("Printing basic blocks links for function \'%s\':\n", m_name.c_str()); - std::vector::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); - bb_itr++) { - printf("ID: %d\t:", (*bb_itr)->bb_id); - if (!(*bb_itr)->predecessor_ids.empty()) { - printf("Predecessors:"); - std::set::iterator p; - for (p = (*bb_itr)->predecessor_ids.begin(); - p != (*bb_itr)->predecessor_ids.end(); p++) { - printf(" %d", *p); + } + printf("\n"); +} + +void function_info::print_basic_block_links() +{ + printf("Printing basic blocks links for function \'%s\':\n", m_name.c_str() ); + std::vector::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { + printf("ID: %d\t:", (*bb_itr)->bb_id); + if ( !(*bb_itr)->predecessor_ids.empty() ) { + printf("Predecessors:"); + std::set::iterator p; + for (p= (*bb_itr)->predecessor_ids.begin();p != (*bb_itr)->predecessor_ids.end();p++) { + printf(" %d", *p); + } + printf("\t"); } - printf("\t"); - } - if (!(*bb_itr)->successor_ids.empty()) { - printf("Successors:"); - std::set::iterator s; - for (s = (*bb_itr)->successor_ids.begin(); - s != (*bb_itr)->successor_ids.end(); s++) { - printf(" %d", *s); + if ( !(*bb_itr)->successor_ids.empty() ) { + printf("Successors:"); + std::set::iterator s; + for (s= (*bb_itr)->successor_ids.begin();s != (*bb_itr)->successor_ids.end();s++) { + printf(" %d", *s); + } } - } - printf("\n"); - } + printf("\n"); + } } -operand_info *function_info::find_break_target( - ptx_instruction *p_break_insn) // find the target of a break instruction +operand_info* function_info::find_break_target( ptx_instruction * p_break_insn ) //find the target of a break instruction { - const basic_block_t *break_bb = p_break_insn->get_bb(); - // go through the dominator tree - for (const basic_block_t *p_bb = break_bb; p_bb->immediatedominator_id != -1; - p_bb = m_basic_blocks[p_bb->immediatedominator_id]) { - // reverse search through instructions in basic block for breakaddr - // instruction - unsigned insn_addr = p_bb->ptx_end->get_m_instr_mem_index(); - while (insn_addr >= p_bb->ptx_begin->get_m_instr_mem_index()) { - ptx_instruction *pI = m_instr_mem[insn_addr]; - insn_addr -= 1; - if (pI == NULL) - continue; // temporary solution for variable size instructions - if (pI->get_opcode() == BREAKADDR_OP) { - return &(pI->dst()); + const basic_block_t *break_bb = p_break_insn->get_bb(); + // go through the dominator tree + for(const basic_block_t *p_bb = break_bb; + p_bb->immediatedominator_id != -1; + p_bb = m_basic_blocks[p_bb->immediatedominator_id]) + { + // reverse search through instructions in basic block for breakaddr instruction + unsigned insn_addr = p_bb->ptx_end->get_m_instr_mem_index(); + while (insn_addr >= p_bb->ptx_begin->get_m_instr_mem_index()) { + ptx_instruction *pI = m_instr_mem[insn_addr]; + insn_addr -= 1; + if (pI == NULL) continue; // temporary solution for variable size instructions + if (pI->get_opcode() == BREAKADDR_OP) { + return &(pI->dst()); + } } - } - } + } - assert(0); + assert(0); - // lazy fallback: just traverse backwards? - for (int insn_addr = p_break_insn->get_m_instr_mem_index(); insn_addr >= 0; - insn_addr--) { - ptx_instruction *pI = m_instr_mem[insn_addr]; - if (pI->get_opcode() == BREAKADDR_OP) { - return &(pI->dst()); - } - } + // lazy fallback: just traverse backwards? + for (int insn_addr = p_break_insn->get_m_instr_mem_index(); + insn_addr >= 0; insn_addr--) + { + ptx_instruction *pI = m_instr_mem[insn_addr]; + if (pI->get_opcode() == BREAKADDR_OP) { + return &(pI->dst()); + } + } - return NULL; + return NULL; } -void function_info::connect_basic_blocks() // iterate across m_basic_blocks of - // function, connecting basic blocks - // together +void function_info::connect_basic_blocks( ) //iterate across m_basic_blocks of function, connecting basic blocks together { - std::vector::iterator bb_itr; - std::vector::iterator bb_target_itr; - basic_block_t *exit_bb = m_basic_blocks.back(); - - // start from first basic block, which we know is the entry point - bb_itr = m_basic_blocks.begin(); - for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); - bb_itr++) { - ptx_instruction *pI = (*bb_itr)->ptx_end; - if ((*bb_itr)->is_exit) // reached last basic block, no successors to link - continue; - if (pI->get_opcode() == RETP_OP || pI->get_opcode() == RET_OP || - pI->get_opcode() == EXIT_OP) { - (*bb_itr)->successor_ids.insert(exit_bb->bb_id); - exit_bb->predecessor_ids.insert((*bb_itr)->bb_id); - if (pI->has_pred()) { - printf("GPGPU-Sim PTX: Warning detected predicated return/exit.\n"); - // if predicated, add link to next block - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - if (next_addr < m_instr_mem_size && m_instr_mem[next_addr]) { - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - (*bb_itr)->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } + std::vector::iterator bb_itr; + std::vector::iterator bb_target_itr; + basic_block_t* exit_bb = m_basic_blocks.back(); + + //start from first basic block, which we know is the entry point + bb_itr = m_basic_blocks.begin(); + for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { + ptx_instruction *pI = (*bb_itr)->ptx_end; + if ((*bb_itr)->is_exit) //reached last basic block, no successors to link + continue; + if (pI->get_opcode() == RETP_OP || pI->get_opcode() == RET_OP || pI->get_opcode() == EXIT_OP ) { + (*bb_itr)->successor_ids.insert(exit_bb->bb_id); + exit_bb->predecessor_ids.insert((*bb_itr)->bb_id); + if( pI->has_pred() ) { + printf("GPGPU-Sim PTX: Warning detected predicated return/exit.\n"); + // if predicated, add link to next block + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + if( next_addr < m_instr_mem_size && m_instr_mem[next_addr] ) { + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + (*bb_itr)->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } + } + continue; + } else if (pI->get_opcode() == BRA_OP) { + //find successor and link that basic_block to this one + operand_info &target = pI->dst(); //get operand, e.g. target name + unsigned addr = labels[ target.name() ]; + ptx_instruction *target_pI = m_instr_mem[addr]; + basic_block_t *target_bb = target_pI->get_bb(); + (*bb_itr)->successor_ids.insert(target_bb->bb_id); + target_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } + + if ( !(pI->get_opcode()==BRA_OP && (!pI->has_pred())) ) { + // if basic block does not end in an unpredicated branch, + // then next basic block is also successor + // (this is better than testing for .uni) + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + (*bb_itr)->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } else + assert(pI->get_opcode() == BRA_OP); + } +} +bool function_info::connect_break_targets() //connecting break instructions with proper targets +{ + std::vector::iterator bb_itr; + std::vector::iterator bb_target_itr; + bool modified = false; + + //start from first basic block, which we know is the entry point + bb_itr = m_basic_blocks.begin(); + for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { + basic_block_t *p_bb = *bb_itr; + ptx_instruction *pI = p_bb->ptx_end; + if (p_bb->is_exit) //reached last basic block, no successors to link + continue; + if (pI->get_opcode() == BREAK_OP) { + // backup existing successor_ids for stability check + std::set orig_successor_ids = p_bb->successor_ids; + + // erase the previous linkage with old successors + for(std::set::iterator succ_ids = p_bb->successor_ids.begin(); succ_ids != p_bb->successor_ids.end(); ++succ_ids) { + basic_block_t *successor_bb = m_basic_blocks[*succ_ids]; + successor_bb->predecessor_ids.erase(p_bb->bb_id); + } + p_bb->successor_ids.clear(); + + //find successor and link that basic_block to this one + //successor of a break is set by an preceeding breakaddr instruction + operand_info *target = find_break_target(pI); + unsigned addr = labels[ target->name() ]; + ptx_instruction *target_pI = m_instr_mem[addr]; + basic_block_t *target_bb = target_pI->get_bb(); + p_bb->successor_ids.insert(target_bb->bb_id); + target_bb->predecessor_ids.insert(p_bb->bb_id); + + if (pI->has_pred()) { + // predicated break - add link to next basic block + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + p_bb->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert(p_bb->bb_id); + } + + modified = modified || (orig_successor_ids != p_bb->successor_ids); } - continue; - } else if (pI->get_opcode() == BRA_OP) { - // find successor and link that basic_block to this one - operand_info &target = pI->dst(); // get operand, e.g. target name - unsigned addr = labels[target.name()]; - ptx_instruction *target_pI = m_instr_mem[addr]; - basic_block_t *target_bb = target_pI->get_bb(); - (*bb_itr)->successor_ids.insert(target_bb->bb_id); - target_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } + } - if (!(pI->get_opcode() == BRA_OP && (!pI->has_pred()))) { - // if basic block does not end in an unpredicated branch, - // then next basic block is also successor - // (this is better than testing for .uni) - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - (*bb_itr)->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } else - assert(pI->get_opcode() == BRA_OP); - } -} -bool function_info::connect_break_targets() // connecting break instructions - // with proper targets + return modified; +} +void function_info::do_pdom() +{ + create_basic_blocks(); + connect_basic_blocks(); + bool modified = false; + do { + find_dominators(); + find_idominators(); + modified = connect_break_targets(); + } while (modified == true); + + if ( g_debug_execution>=50 ) { + print_basic_blocks(); + print_basic_block_links(); + print_basic_block_dot(); + } + if ( g_debug_execution>=2 ) { + print_dominators(); + } + find_postdominators(); + find_ipostdominators(); + if ( g_debug_execution>=50 ) { + print_postdominators(); + print_ipostdominators(); + } + printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", m_name.c_str() ); + for ( unsigned ii=0; ii < m_n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions + ptx_instruction *pI = m_instr_mem[ii]; + pI->pre_decode(); + } + printf("GPGPU-Sim PTX: ... done pre-decoding instructions for \'%s\'.\n", m_name.c_str() ); + fflush(stdout); + m_assembled = true; +} +void intersect( std::set &A, const std::set &B ) { - std::vector::iterator bb_itr; - std::vector::iterator bb_target_itr; - bool modified = false; - - // start from first basic block, which we know is the entry point - bb_itr = m_basic_blocks.begin(); - for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); - bb_itr++) { - basic_block_t *p_bb = *bb_itr; - ptx_instruction *pI = p_bb->ptx_end; - if (p_bb->is_exit) // reached last basic block, no successors to link - continue; - if (pI->get_opcode() == BREAK_OP) { - // backup existing successor_ids for stability check - std::set orig_successor_ids = p_bb->successor_ids; - - // erase the previous linkage with old successors - for (std::set::iterator succ_ids = p_bb->successor_ids.begin(); - succ_ids != p_bb->successor_ids.end(); ++succ_ids) { - basic_block_t *successor_bb = m_basic_blocks[*succ_ids]; - successor_bb->predecessor_ids.erase(p_bb->bb_id); + // return intersection of A and B in A + for( std::set::iterator a=A.begin(); a!=A.end(); ) { + std::set::iterator a_next = a; + a_next++; + if( B.find(*a) == B.end() ) { + A.erase(*a); + a = a_next; + } else + a++; + } +} + +bool is_equal( const std::set &A, const std::set &B ) +{ + if( A.size() != B.size() ) + return false; + for( std::set::iterator b=B.begin(); b!=B.end(); b++ ) + if( A.find(*b) == A.end() ) + return false; + return true; +} + +void print_set(const std::set &A) +{ + std::set::iterator a; + for (a= A.begin(); a != A.end(); a++) { + printf("%d ", (*a)); + } + printf("\n"); +} + +void function_info::find_dominators( ) +{ + // find dominators using algorithm of Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 + printf("GPGPU-Sim PTX: Finding dominators for \'%s\'...\n", m_name.c_str() ); + fflush(stdout); + assert( m_basic_blocks.size() >= 2 ); // must have a distinquished entry block + std::vector::iterator bb_itr = m_basic_blocks.begin(); + (*bb_itr)->dominator_ids.insert((*bb_itr)->bb_id); // the only dominator of the entry block is the entry + //copy all basic blocks to all dominator lists EXCEPT for the entry block + for (++bb_itr;bb_itr != m_basic_blocks.end(); bb_itr++) { + for (unsigned i = 0; i < m_basic_blocks.size(); i++) + (*bb_itr)->dominator_ids.insert(i); + } + bool change = true; + while (change) { + change = false; + for ( int h = 1/*skip entry*/; h < m_basic_blocks.size(); ++h ) { + assert( m_basic_blocks[h]->bb_id == (unsigned)h ); + std::set T; + for (unsigned i=0;i< m_basic_blocks.size();i++) + T.insert(i); + for ( std::set::iterator s = m_basic_blocks[h]->predecessor_ids.begin();s != m_basic_blocks[h]->predecessor_ids.end();s++) + intersect(T, m_basic_blocks[*s]->dominator_ids); + T.insert(h); + if (!is_equal(T, m_basic_blocks[h]->dominator_ids)) { + change = true; + m_basic_blocks[h]->dominator_ids = T; + } } - p_bb->successor_ids.clear(); - - // find successor and link that basic_block to this one - // successor of a break is set by an preceeding breakaddr instruction - operand_info *target = find_break_target(pI); - unsigned addr = labels[target->name()]; - ptx_instruction *target_pI = m_instr_mem[addr]; - basic_block_t *target_bb = target_pI->get_bb(); - p_bb->successor_ids.insert(target_bb->bb_id); - target_bb->predecessor_ids.insert(p_bb->bb_id); - - if (pI->has_pred()) { - // predicated break - add link to next basic block - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - p_bb->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert(p_bb->bb_id); + } + //clean the basic block of dominators of it has no predecessors -- except for entry block + bb_itr = m_basic_blocks.begin(); + for (++bb_itr;bb_itr != m_basic_blocks.end(); bb_itr++) { + if ((*bb_itr)->predecessor_ids.empty()) + (*bb_itr)->dominator_ids.clear(); + } +} + +void function_info::find_postdominators( ) +{ + // find postdominators using algorithm of Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 + printf("GPGPU-Sim PTX: Finding postdominators for \'%s\'...\n", m_name.c_str() ); + fflush(stdout); + assert( m_basic_blocks.size() >= 2 ); // must have a distinquished exit block + std::vector::reverse_iterator bb_itr = m_basic_blocks.rbegin(); + (*bb_itr)->postdominator_ids.insert((*bb_itr)->bb_id); // the only postdominator of the exit block is the exit + for (++bb_itr;bb_itr != m_basic_blocks.rend();bb_itr++) { //copy all basic blocks to all postdominator lists EXCEPT for the exit block + for (unsigned i=0; ipostdominator_ids.insert(i); + } + bool change = true; + while (change) { + change = false; + for ( int h = m_basic_blocks.size()-2/*skip exit*/; h >= 0 ; --h ) { + assert( m_basic_blocks[h]->bb_id == (unsigned)h ); + std::set T; + for (unsigned i=0;i< m_basic_blocks.size();i++) + T.insert(i); + for ( std::set::iterator s = m_basic_blocks[h]->successor_ids.begin();s != m_basic_blocks[h]->successor_ids.end();s++) + intersect(T, m_basic_blocks[*s]->postdominator_ids); + T.insert(h); + if (!is_equal(T,m_basic_blocks[h]->postdominator_ids)) { + change = true; + m_basic_blocks[h]->postdominator_ids = T; + } } + } +} - modified = modified || (orig_successor_ids != p_bb->successor_ids); - } - } - - return modified; -} -void function_info::do_pdom() { - create_basic_blocks(); - connect_basic_blocks(); - bool modified = false; - do { - find_dominators(); - find_idominators(); - modified = connect_break_targets(); - } while (modified == true); - - if (g_debug_execution >= 50) { - print_basic_blocks(); - print_basic_block_links(); - print_basic_block_dot(); - } - if (g_debug_execution >= 2) { - print_dominators(); - } - find_postdominators(); - find_ipostdominators(); - if (g_debug_execution >= 50) { - print_postdominators(); - print_ipostdominators(); - } - printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", - m_name.c_str()); - for (unsigned ii = 0; ii < m_n; - ii += m_instr_mem[ii]->inst_size()) { // handle branch instructions - ptx_instruction *pI = m_instr_mem[ii]; - pI->pre_decode(); - } - printf("GPGPU-Sim PTX: ... done pre-decoding instructions for \'%s\'.\n", - m_name.c_str()); - fflush(stdout); - m_assembled = true; -} -void intersect(std::set &A, const std::set &B) { - // return intersection of A and B in A - for (std::set::iterator a = A.begin(); a != A.end();) { - std::set::iterator a_next = a; - a_next++; - if (B.find(*a) == B.end()) { - A.erase(*a); - a = a_next; - } else - a++; - } -} - -bool is_equal(const std::set &A, const std::set &B) { - if (A.size() != B.size()) return false; - for (std::set::iterator b = B.begin(); b != B.end(); b++) - if (A.find(*b) == A.end()) return false; - return true; -} - -void print_set(const std::set &A) { - std::set::iterator a; - for (a = A.begin(); a != A.end(); a++) { - printf("%d ", (*a)); - } - printf("\n"); -} - -void function_info::find_dominators() { - // find dominators using algorithm of Muchnick's Adv. Compiler Design & - // Implemmntation Fig 7.14 - printf("GPGPU-Sim PTX: Finding dominators for \'%s\'...\n", m_name.c_str()); - fflush(stdout); - assert(m_basic_blocks.size() >= 2); // must have a distinquished entry block - std::vector::iterator bb_itr = m_basic_blocks.begin(); - (*bb_itr)->dominator_ids.insert( - (*bb_itr)->bb_id); // the only dominator of the entry block is the entry - // copy all basic blocks to all dominator lists EXCEPT for the entry block - for (++bb_itr; bb_itr != m_basic_blocks.end(); bb_itr++) { - for (unsigned i = 0; i < m_basic_blocks.size(); i++) - (*bb_itr)->dominator_ids.insert(i); - } - bool change = true; - while (change) { - change = false; - for (int h = 1 /*skip entry*/; h < m_basic_blocks.size(); ++h) { - assert(m_basic_blocks[h]->bb_id == (unsigned)h); - std::set T; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) T.insert(i); - for (std::set::iterator s = - m_basic_blocks[h]->predecessor_ids.begin(); - s != m_basic_blocks[h]->predecessor_ids.end(); s++) - intersect(T, m_basic_blocks[*s]->dominator_ids); - T.insert(h); - if (!is_equal(T, m_basic_blocks[h]->dominator_ids)) { - change = true; - m_basic_blocks[h]->dominator_ids = T; +void function_info::find_ipostdominators( ) +{ + // find immediate postdominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + printf("GPGPU-Sim PTX: Finding immediate postdominators for \'%s\'...\n", m_name.c_str() ); + fflush(stdout); + assert( m_basic_blocks.size() >= 2 ); // must have a distinquished exit block + for (unsigned i=0; iTmp_ids = m_basic_blocks[i]->postdominator_ids; + assert( m_basic_blocks[i]->bb_id == i ); + m_basic_blocks[i]->Tmp_ids.erase(i); + } + for ( int n = m_basic_blocks.size()-2; n >=0;--n) { + // point iterator to basic block before the exit + for( std::set::iterator s=m_basic_blocks[n]->Tmp_ids.begin(); s != m_basic_blocks[n]->Tmp_ids.end(); s++ ) { + int bb_s = *s; + for( std::set::iterator t=m_basic_blocks[n]->Tmp_ids.begin(); t != m_basic_blocks[n]->Tmp_ids.end(); ) { + std::set::iterator t_next = t; t_next++; // might erase thing pointed to be t, invalidating iterator t + if( *s == *t ) { + t = t_next; + continue; + } + int bb_t = *t; + if( m_basic_blocks[bb_s]->postdominator_ids.find(bb_t) != m_basic_blocks[bb_s]->postdominator_ids.end() ) + m_basic_blocks[n]->Tmp_ids.erase(bb_t); + t = t_next; + } } - } - } - // clean the basic block of dominators of it has no predecessors -- except for - // entry block - bb_itr = m_basic_blocks.begin(); - for (++bb_itr; bb_itr != m_basic_blocks.end(); bb_itr++) { - if ((*bb_itr)->predecessor_ids.empty()) (*bb_itr)->dominator_ids.clear(); - } -} - -void function_info::find_postdominators() { - // find postdominators using algorithm of Muchnick's Adv. Compiler Design & - // Implemmntation Fig 7.14 - printf("GPGPU-Sim PTX: Finding postdominators for \'%s\'...\n", - m_name.c_str()); - fflush(stdout); - assert(m_basic_blocks.size() >= 2); // must have a distinquished exit block - std::vector::reverse_iterator bb_itr = - m_basic_blocks.rbegin(); - (*bb_itr)->postdominator_ids.insert( - (*bb_itr) - ->bb_id); // the only postdominator of the exit block is the exit - for (++bb_itr; bb_itr != m_basic_blocks.rend(); - bb_itr++) { // copy all basic blocks to all postdominator lists EXCEPT - // for the exit block - for (unsigned i = 0; i < m_basic_blocks.size(); i++) - (*bb_itr)->postdominator_ids.insert(i); - } - bool change = true; - while (change) { - change = false; - for (int h = m_basic_blocks.size() - 2 /*skip exit*/; h >= 0; --h) { - assert(m_basic_blocks[h]->bb_id == (unsigned)h); - std::set T; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) T.insert(i); - for (std::set::iterator s = m_basic_blocks[h]->successor_ids.begin(); - s != m_basic_blocks[h]->successor_ids.end(); s++) - intersect(T, m_basic_blocks[*s]->postdominator_ids); - T.insert(h); - if (!is_equal(T, m_basic_blocks[h]->postdominator_ids)) { - change = true; - m_basic_blocks[h]->postdominator_ids = T; + } + unsigned num_ipdoms=0; + for ( int n = m_basic_blocks.size()-1; n >=0;--n) { + assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); + // if the above assert fails we have an error in either postdominator + // computation, the flow graph does not have a unique exit, or some other error + if( !m_basic_blocks[n]->Tmp_ids.empty() ) { + m_basic_blocks[n]->immediatepostdominator_id = *m_basic_blocks[n]->Tmp_ids.begin(); + num_ipdoms++; } - } - } -} - -void function_info::find_ipostdominators() { - // find immediate postdominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - printf("GPGPU-Sim PTX: Finding immediate postdominators for \'%s\'...\n", - m_name.c_str()); - fflush(stdout); - assert(m_basic_blocks.size() >= 2); // must have a distinquished exit block - for (unsigned i = 0; i < m_basic_blocks.size(); - i++) { // initialize Tmp(n) to all pdoms of n except for n - m_basic_blocks[i]->Tmp_ids = m_basic_blocks[i]->postdominator_ids; - assert(m_basic_blocks[i]->bb_id == i); - m_basic_blocks[i]->Tmp_ids.erase(i); - } - for (int n = m_basic_blocks.size() - 2; n >= 0; --n) { - // point iterator to basic block before the exit - for (std::set::iterator s = m_basic_blocks[n]->Tmp_ids.begin(); - s != m_basic_blocks[n]->Tmp_ids.end(); s++) { - int bb_s = *s; - for (std::set::iterator t = m_basic_blocks[n]->Tmp_ids.begin(); - t != m_basic_blocks[n]->Tmp_ids.end();) { - std::set::iterator t_next = t; - t_next++; // might erase thing pointed to be t, invalidating iterator t - if (*s == *t) { - t = t_next; - continue; - } - int bb_t = *t; - if (m_basic_blocks[bb_s]->postdominator_ids.find(bb_t) != - m_basic_blocks[bb_s]->postdominator_ids.end()) - m_basic_blocks[n]->Tmp_ids.erase(bb_t); - t = t_next; + } + assert( num_ipdoms == m_basic_blocks.size()-1 ); + // the exit node does not have an immediate post dominator, but everyone else should +} + +void function_info::find_idominators( ) +{ + // find immediate dominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + printf("GPGPU-Sim PTX: Finding immediate dominators for \'%s\'...\n", m_name.c_str() ); + fflush(stdout); + assert( m_basic_blocks.size() >= 2 ); // must have a distinquished entry block + for (unsigned i=0; iTmp_ids = m_basic_blocks[i]->dominator_ids; + assert( m_basic_blocks[i]->bb_id == i ); + m_basic_blocks[i]->Tmp_ids.erase(i); + } + for ( int n = 0; n < m_basic_blocks.size(); ++n) { + // point iterator to basic block before the exit + for( std::set::iterator s=m_basic_blocks[n]->Tmp_ids.begin(); s != m_basic_blocks[n]->Tmp_ids.end(); s++ ) { + int bb_s = *s; + for( std::set::iterator t=m_basic_blocks[n]->Tmp_ids.begin(); t != m_basic_blocks[n]->Tmp_ids.end(); ) { + std::set::iterator t_next = t; t_next++; // might erase thing pointed to be t, invalidating iterator t + if( *s == *t ) { + t = t_next; + continue; + } + int bb_t = *t; + if( m_basic_blocks[bb_s]->dominator_ids.find(bb_t) != m_basic_blocks[bb_s]->dominator_ids.end() ) + m_basic_blocks[n]->Tmp_ids.erase(bb_t); + t = t_next; + } } - } - } - unsigned num_ipdoms = 0; - for (int n = m_basic_blocks.size() - 1; n >= 0; --n) { - assert(m_basic_blocks[n]->Tmp_ids.size() <= 1); - // if the above assert fails we have an error in either postdominator - // computation, the flow graph does not have a unique exit, or some other - // error - if (!m_basic_blocks[n]->Tmp_ids.empty()) { - m_basic_blocks[n]->immediatepostdominator_id = - *m_basic_blocks[n]->Tmp_ids.begin(); - num_ipdoms++; - } - } - assert(num_ipdoms == m_basic_blocks.size() - 1); - // the exit node does not have an immediate post dominator, but everyone else - // should -} - -void function_info::find_idominators() { - // find immediate dominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - printf("GPGPU-Sim PTX: Finding immediate dominators for \'%s\'...\n", - m_name.c_str()); - fflush(stdout); - assert(m_basic_blocks.size() >= 2); // must have a distinquished entry block - for (unsigned i = 0; i < m_basic_blocks.size(); - i++) { // initialize Tmp(n) to all doms of n except for n - m_basic_blocks[i]->Tmp_ids = m_basic_blocks[i]->dominator_ids; - assert(m_basic_blocks[i]->bb_id == i); - m_basic_blocks[i]->Tmp_ids.erase(i); - } - for (int n = 0; n < m_basic_blocks.size(); ++n) { - // point iterator to basic block before the exit - for (std::set::iterator s = m_basic_blocks[n]->Tmp_ids.begin(); - s != m_basic_blocks[n]->Tmp_ids.end(); s++) { - int bb_s = *s; - for (std::set::iterator t = m_basic_blocks[n]->Tmp_ids.begin(); - t != m_basic_blocks[n]->Tmp_ids.end();) { - std::set::iterator t_next = t; - t_next++; // might erase thing pointed to be t, invalidating iterator t - if (*s == *t) { - t = t_next; - continue; - } - int bb_t = *t; - if (m_basic_blocks[bb_s]->dominator_ids.find(bb_t) != - m_basic_blocks[bb_s]->dominator_ids.end()) - m_basic_blocks[n]->Tmp_ids.erase(bb_t); - t = t_next; + } + unsigned num_idoms=0; + unsigned num_nopred = 0; + for ( int n = 0; n < m_basic_blocks.size(); ++n) { + //assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); + // if the above assert fails we have an error in either dominator + // computation, the flow graph does not have a unique entry, or some other error + if( !m_basic_blocks[n]->Tmp_ids.empty() ) { + m_basic_blocks[n]->immediatedominator_id = *m_basic_blocks[n]->Tmp_ids.begin(); + num_idoms++; + } else if (m_basic_blocks[n]->predecessor_ids.empty()) { + num_nopred += 1; } - } - } - unsigned num_idoms = 0; - unsigned num_nopred = 0; - for (int n = 0; n < m_basic_blocks.size(); ++n) { - // assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); - // if the above assert fails we have an error in either dominator - // computation, the flow graph does not have a unique entry, or some other - // error - if (!m_basic_blocks[n]->Tmp_ids.empty()) { - m_basic_blocks[n]->immediatedominator_id = - *m_basic_blocks[n]->Tmp_ids.begin(); - num_idoms++; - } else if (m_basic_blocks[n]->predecessor_ids.empty()) { - num_nopred += 1; - } - } - assert(num_idoms == m_basic_blocks.size() - num_nopred); - // the entry node does not have an immediate dominator, but everyone else - // should -} - -void function_info::print_dominators() { - printf("Printing dominators for function \'%s\':\n", m_name.c_str()); - std::vector::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - for (std::set::iterator j = m_basic_blocks[i]->dominator_ids.begin(); - j != m_basic_blocks[i]->dominator_ids.end(); j++) - printf(" %d", *j); - printf("\n"); - } -} - -void function_info::print_postdominators() { - printf("Printing postdominators for function \'%s\':\n", m_name.c_str()); - std::vector::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - for (std::set::iterator j = - m_basic_blocks[i]->postdominator_ids.begin(); - j != m_basic_blocks[i]->postdominator_ids.end(); j++) - printf(" %d", *j); - printf("\n"); - } -} - -void function_info::print_ipostdominators() { - printf("Printing immediate postdominators for function \'%s\':\n", - m_name.c_str()); - std::vector::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - printf("%d\n", m_basic_blocks[i]->immediatepostdominator_id); - } -} - -void function_info::print_idominators() { - printf("Printing immediate dominators for function \'%s\':\n", - m_name.c_str()); - std::vector::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - printf("%d\n", m_basic_blocks[i]->immediatedominator_id); - } -} - -unsigned function_info::get_num_reconvergence_pairs() { - if (!num_reconvergence_pairs) { - if (m_basic_blocks.size() == 0) return 0; - for (unsigned i = 0; i < (m_basic_blocks.size() - 1); - i++) { // last basic block containing exit obviously won't have a pair - if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { - num_reconvergence_pairs++; + } + assert( num_idoms == m_basic_blocks.size()-num_nopred ); + // the entry node does not have an immediate dominator, but everyone else should +} + +void function_info::print_dominators() +{ + printf("Printing dominators for function \'%s\':\n", m_name.c_str() ); + std::vector::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + for( std::set::iterator j=m_basic_blocks[i]->dominator_ids.begin(); j!=m_basic_blocks[i]->dominator_ids.end(); j++) + printf(" %d", *j ); + printf("\n"); + } +} + +void function_info::print_postdominators() +{ + printf("Printing postdominators for function \'%s\':\n", m_name.c_str() ); + std::vector::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + for( std::set::iterator j=m_basic_blocks[i]->postdominator_ids.begin(); j!=m_basic_blocks[i]->postdominator_ids.end(); j++) + printf(" %d", *j ); + printf("\n"); + } +} + +void function_info::print_ipostdominators() +{ + printf("Printing immediate postdominators for function \'%s\':\n", m_name.c_str() ); + std::vector::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + printf("%d\n", m_basic_blocks[i]->immediatepostdominator_id); + } +} + +void function_info::print_idominators() +{ + printf("Printing immediate dominators for function \'%s\':\n", m_name.c_str() ); + std::vector::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + printf("%d\n", m_basic_blocks[i]->immediatedominator_id); + } +} + +unsigned function_info::get_num_reconvergence_pairs() +{ + if (!num_reconvergence_pairs) { + if( m_basic_blocks.size() == 0 ) + return 0; + for (unsigned i=0; i< (m_basic_blocks.size()-1); i++) { //last basic block containing exit obviously won't have a pair + if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { + num_reconvergence_pairs++; + } } - } - } - return num_reconvergence_pairs; + } + return num_reconvergence_pairs; } -void function_info::get_reconvergence_pairs(gpgpu_recon_t *recon_points) { - unsigned idx = 0; // array index - if (m_basic_blocks.size() == 0) return; - for (unsigned i = 0; i < (m_basic_blocks.size() - 1); - i++) { // last basic block containing exit obviously won't have a pair +void function_info::get_reconvergence_pairs(gpgpu_recon_t* recon_points) +{ + unsigned idx=0; //array index + if( m_basic_blocks.size() == 0 ) + return; + for (unsigned i=0; i< (m_basic_blocks.size()-1); i++) { //last basic block containing exit obviously won't have a pair #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("i=%d\n", i); - fflush(stdout); + printf("i=%d\n", i); fflush(stdout); #endif - if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { + if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("\tbranch!\n"); - printf("\tbb_id=%d; ipdom=%d\n", m_basic_blocks[i]->bb_id, - m_basic_blocks[i]->immediatepostdominator_id); - printf("\tm_instr_mem index=%d\n", - m_basic_blocks[i]->ptx_end->get_m_instr_mem_index()); - fflush(stdout); + printf("\tbranch!\n"); + printf("\tbb_id=%d; ipdom=%d\n", m_basic_blocks[i]->bb_id, m_basic_blocks[i]->immediatepostdominator_id); + printf("\tm_instr_mem index=%d\n", m_basic_blocks[i]->ptx_end->get_m_instr_mem_index()); + fflush(stdout); #endif - recon_points[idx].source_pc = m_basic_blocks[i]->ptx_end->get_PC(); - recon_points[idx].source_inst = m_basic_blocks[i]->ptx_end; + recon_points[idx].source_pc = m_basic_blocks[i]->ptx_end->get_PC(); + recon_points[idx].source_inst = m_basic_blocks[i]->ptx_end; #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("\trecon_points[idx].source_pc=%d\n", recon_points[idx].source_pc); + printf("\trecon_points[idx].source_pc=%d\n", recon_points[idx].source_pc); #endif - if (m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] - ->ptx_begin) { - recon_points[idx].target_pc = - m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] - ->ptx_begin->get_PC(); - recon_points[idx].target_inst = - m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] - ->ptx_begin; - } else { - // reconverge after function return - recon_points[idx].target_pc = -2; - recon_points[idx].target_inst = NULL; - } + if( m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin ) { + recon_points[idx].target_pc = m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin->get_PC(); + recon_points[idx].target_inst = m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin; + } else { + // reconverge after function return + recon_points[idx].target_pc = -2; + recon_points[idx].target_inst = NULL; + } #ifdef DEBUG_GET_RECONVERG_PAIRS - m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] - ->ptx_begin->print_insn(); - printf("\trecon_points[idx].target_pc=%d\n", recon_points[idx].target_pc); - fflush(stdout); + m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin->print_insn(); + printf("\trecon_points[idx].target_pc=%d\n", recon_points[idx].target_pc); fflush(stdout); #endif - idx++; - } - } + idx++; + } + } } // interface with graphviz (print the graph in DOT language) for plotting -void function_info::print_basic_block_dot() { - printf("Basic Block in DOT\n"); - printf("digraph %s {\n", m_name.c_str()); - std::vector::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); - bb_itr++) { - printf("\t"); - std::set::iterator s; - for (s = (*bb_itr)->successor_ids.begin(); - s != (*bb_itr)->successor_ids.end(); s++) { - unsigned succ_bb = *s; - printf("%d -> %d; ", (*bb_itr)->bb_id, succ_bb); - } - printf("\n"); - } - printf("}\n"); -} - -unsigned ptx_kernel_shmem_size(void *kernel_impl) { - function_info *f = (function_info *)kernel_impl; - const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); - return kernel_info->smem; -} - -unsigned ptx_kernel_nregs(void *kernel_impl) { - function_info *f = (function_info *)kernel_impl; - const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); - return kernel_info->regs; -} - -unsigned type_info_key::type_decode(size_t &size, int &basic_type) const { - int type = scalar_type(); - return type_decode(type, size, basic_type); -} - -unsigned type_info_key::type_decode(int type, size_t &size, int &basic_type) { - switch (type) { - case S8_TYPE: - size = 8; - basic_type = 1; - return 0; - case S16_TYPE: - size = 16; - basic_type = 1; - return 1; - case S32_TYPE: - size = 32; - basic_type = 1; - return 2; - case S64_TYPE: - size = 64; - basic_type = 1; - return 3; - case U8_TYPE: - size = 8; - basic_type = 0; - return 4; - case U16_TYPE: - size = 16; - basic_type = 0; - return 5; - case U32_TYPE: - size = 32; - basic_type = 0; - return 6; - case U64_TYPE: - size = 64; - basic_type = 0; - return 7; - case F16_TYPE: - size = 16; - basic_type = -1; - return 8; - case F32_TYPE: - size = 32; - basic_type = -1; - return 9; - case F64_TYPE: - size = 64; - basic_type = -1; - return 10; - case FF64_TYPE: - size = 64; - basic_type = -1; - return 10; - case PRED_TYPE: - size = 1; - basic_type = 2; - return 11; - case B8_TYPE: - size = 8; - basic_type = 0; - return 12; - case B16_TYPE: - size = 16; - basic_type = 0; - return 13; - case B32_TYPE: - size = 32; - basic_type = 0; - return 14; - case B64_TYPE: - size = 64; - basic_type = 0; - return 15; - case BB64_TYPE: - size = 64; - basic_type = 0; - return 15; - case BB128_TYPE: - size = 128; - basic_type = 0; - return 16; - case TEXREF_TYPE: - case SAMPLERREF_TYPE: - case SURFREF_TYPE: - size = 32; - basic_type = 3; - return 16; - default: - printf("ERROR ** type_decode() does not know about \"%s\"\n", - decode_token(type)); - assert(0); - return 0xDEADBEEF; - } -} - -arg_buffer_t copy_arg_to_buffer(ptx_thread_info *thread, - operand_info actual_param_op, - const symbol *formal_param) { - if (actual_param_op.is_reg()) { - ptx_reg_t value = thread->get_reg(actual_param_op.get_symbol()); - return arg_buffer_t(formal_param, actual_param_op, value); - } else if (actual_param_op.is_param_local()) { - unsigned size = formal_param->get_size_in_bytes(); - addr_t frame_offset = actual_param_op.get_symbol()->get_address(); - addr_t from_addr = thread->get_local_mem_stack_pointer() + frame_offset; - char buffer[1024]; - assert(size < 1024); - thread->m_local_mem->read(from_addr, size, buffer); - return arg_buffer_t(formal_param, actual_param_op, buffer, size); - } else { - printf( - "GPGPU-Sim PTX: ERROR ** need to add support for this operand type in " - "call/return\n"); - abort(); - } -} - -void copy_args_into_buffer_list(const ptx_instruction *pI, - ptx_thread_info *thread, - const function_info *target_func, - arg_buffer_list_t &arg_values) { - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - for (unsigned arg = 0; arg < n_args; arg++) { - const operand_info &actual_param_op = - pI->operand_lookup(n_return + 1 + arg); - const symbol *formal_param = target_func->get_arg(arg); - arg_values.push_back( - copy_arg_to_buffer(thread, actual_param_op, formal_param)); - } -} - -void copy_buffer_to_frame(ptx_thread_info *thread, const arg_buffer_t &a) { - if (a.is_reg()) { - ptx_reg_t value = a.get_reg(); - operand_info dst_reg = - operand_info(a.get_dst(), thread->get_gpu()->gpgpu_ctx); - thread->set_reg(dst_reg.get_symbol(), value); - } else { - const void *buffer = a.get_param_buffer(); - size_t size = a.get_param_buffer_size(); - const symbol *dst = a.get_dst(); - addr_t frame_offset = dst->get_address(); - addr_t to_addr = thread->get_local_mem_stack_pointer() + frame_offset; - thread->m_local_mem->write(to_addr, size, buffer, NULL, NULL); - } -} - -void copy_buffer_list_into_frame(ptx_thread_info *thread, - arg_buffer_list_t &arg_values) { - arg_buffer_list_t::iterator a; - for (a = arg_values.begin(); a != arg_values.end(); a++) { - copy_buffer_to_frame(thread, *a); - } -} - -static std::list check_operands( - int opcode, const std::list &scalar_type, - const std::list &operands, gpgpu_context *ctx) { - static int g_warn_literal_operands_two_type_inst; - if ((opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || - (opcode == TEX_OP) || (opcode == MMA_OP) || (opcode == DP4A_OP)) { - // just make sure these do not have have const operands... - if (!g_warn_literal_operands_two_type_inst) { - std::list::const_iterator o; - for (o = operands.begin(); o != operands.end(); o++) { - const operand_info &op = *o; - if (op.is_literal()) { - printf( - "GPGPU-Sim PTX: PTX uses two scalar type intruction with literal " - "operand.\n"); - g_warn_literal_operands_two_type_inst = 1; - } +void function_info::print_basic_block_dot() +{ + printf("Basic Block in DOT\n"); + printf("digraph %s {\n", m_name.c_str()); + std::vector::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { + printf("\t"); + std::set::iterator s; + for (s = (*bb_itr)->successor_ids.begin();s != (*bb_itr)->successor_ids.end();s++) { + unsigned succ_bb = *s; + printf("%d -> %d; ", (*bb_itr)->bb_id, succ_bb ); } - } - } else { - assert(scalar_type.size() < 2); - if (scalar_type.size() == 1) { - std::list result; - int inst_type = scalar_type.front(); - std::list::const_iterator o; - for (o = operands.begin(); o != operands.end(); o++) { - const operand_info &op = *o; - if (op.is_literal()) { - if ((op.get_type() == double_op_t) && (inst_type == F32_TYPE)) { - ptx_reg_t v = op.get_literal_value(); - float u = (float)v.f64; - operand_info n(u, ctx); - result.push_back(n); - } else { - result.push_back(op); - } - } else { - result.push_back(op); + printf("\n"); + } + printf("}\n"); +} + +unsigned ptx_kernel_shmem_size( void *kernel_impl ) +{ + function_info *f = (function_info*)kernel_impl; + const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); + return kernel_info->smem; +} + +unsigned ptx_kernel_nregs( void *kernel_impl ) +{ + function_info *f = (function_info*)kernel_impl; + const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); + return kernel_info->regs; +} + +unsigned type_info_key::type_decode( size_t &size, int &basic_type ) const +{ + int type = scalar_type(); + return type_decode(type,size,basic_type); +} + +unsigned type_info_key::type_decode( int type, size_t &size, int &basic_type ) +{ + switch ( type ) { + case S8_TYPE: size=8; basic_type=1; return 0; + case S16_TYPE: size=16; basic_type=1; return 1; + case S32_TYPE: size=32; basic_type=1; return 2; + case S64_TYPE: size=64; basic_type=1; return 3; + case U8_TYPE: size=8; basic_type=0; return 4; + case U16_TYPE: size=16; basic_type=0; return 5; + case U32_TYPE: size=32; basic_type=0; return 6; + case U64_TYPE: size=64; basic_type=0; return 7; + case F16_TYPE: size=16; basic_type=-1; return 8; + case F32_TYPE: size=32; basic_type=-1; return 9; + case F64_TYPE: size=64; basic_type=-1; return 10; + case FF64_TYPE: size=64; basic_type=-1; return 10; + case PRED_TYPE: size=1; basic_type=2; return 11; + case B8_TYPE: size=8; basic_type=0; return 12; + case B16_TYPE: size=16; basic_type=0; return 13; + case B32_TYPE: size=32; basic_type=0; return 14; + case B64_TYPE: size=64; basic_type=0; return 15; + case BB64_TYPE: size=64; basic_type=0; return 15; + case BB128_TYPE: size=128; basic_type=0; return 16; + case TEXREF_TYPE: case SAMPLERREF_TYPE: case SURFREF_TYPE: + size=32; basic_type=3; return 16; + default: + printf("ERROR ** type_decode() does not know about \"%s\"\n", decode_token(type) ); + assert(0); + return 0xDEADBEEF; + } +} + +arg_buffer_t copy_arg_to_buffer(ptx_thread_info * thread, operand_info actual_param_op, const symbol * formal_param) +{ + if( actual_param_op.is_reg() ) { + ptx_reg_t value = thread->get_reg(actual_param_op.get_symbol()); + return arg_buffer_t(formal_param,actual_param_op,value); + } else if ( actual_param_op.is_param_local() ) { + unsigned size=formal_param->get_size_in_bytes(); + addr_t frame_offset = actual_param_op.get_symbol()->get_address(); + addr_t from_addr = thread->get_local_mem_stack_pointer() + frame_offset; + char buffer[1024]; + assert(size<1024); + thread->m_local_mem->read(from_addr,size,buffer); + return arg_buffer_t(formal_param,actual_param_op,buffer,size); + } else { + printf("GPGPU-Sim PTX: ERROR ** need to add support for this operand type in call/return\n"); + abort(); + } +} + +void copy_args_into_buffer_list( const ptx_instruction * pI, + ptx_thread_info * thread, + const function_info * target_func, + arg_buffer_list_t &arg_values ) +{ + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + for( unsigned arg=0; arg < n_args; arg ++ ) { + const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); + const symbol *formal_param = target_func->get_arg(arg); + arg_values.push_back( copy_arg_to_buffer(thread, actual_param_op, formal_param) ); + } +} + +void copy_buffer_to_frame(ptx_thread_info * thread, const arg_buffer_t &a) +{ + if( a.is_reg() ) { + ptx_reg_t value = a.get_reg(); + operand_info dst_reg = operand_info(a.get_dst(), thread->get_gpu()->gpgpu_ctx); + thread->set_reg(dst_reg.get_symbol(),value); + } else { + const void *buffer = a.get_param_buffer(); + size_t size = a.get_param_buffer_size(); + const symbol *dst = a.get_dst(); + addr_t frame_offset = dst->get_address(); + addr_t to_addr = thread->get_local_mem_stack_pointer() + frame_offset; + thread->m_local_mem->write(to_addr,size,buffer,NULL,NULL); + } +} + +void copy_buffer_list_into_frame(ptx_thread_info * thread, arg_buffer_list_t &arg_values) +{ + arg_buffer_list_t::iterator a; + for( a=arg_values.begin(); a != arg_values.end(); a++ ) { + copy_buffer_to_frame(thread, *a); + } +} + + + +static std::list check_operands( int opcode, + const std::list &scalar_type, + const std::list &operands, + gpgpu_context* ctx) +{ + static int g_warn_literal_operands_two_type_inst; + if( (opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || (opcode == TEX_OP) || (opcode==MMA_OP) || (opcode == DP4A_OP)) { + // just make sure these do not have have const operands... + if( !g_warn_literal_operands_two_type_inst ) { + std::list::const_iterator o; + for( o = operands.begin(); o != operands.end(); o++ ) { + const operand_info &op = *o; + if( op.is_literal() ) { + printf("GPGPU-Sim PTX: PTX uses two scalar type intruction with literal operand.\n"); + g_warn_literal_operands_two_type_inst = 1; + } + } } - } - return result; - } - } - return operands; -} - -ptx_instruction::ptx_instruction( - int opcode, const symbol *pred, int neg_pred, int pred_mod, symbol *label, - const std::list &operands, const operand_info &return_var, - const std::list &options, const std::list &wmma_options, - const std::list &scalar_type, memory_space_t space_spec, - const char *file, unsigned line, const char *source, - const core_config *config, gpgpu_context *ctx) - : warp_inst_t(config), m_return_var(ctx) { - gpgpu_ctx = ctx; - m_uid = ++(ctx->g_num_ptx_inst_uid); - m_PC = 0; - m_opcode = opcode; - m_pred = pred; - m_neg_pred = neg_pred; - m_pred_mod = pred_mod; - m_label = label; - const std::list checked_operands = - check_operands(opcode, scalar_type, operands, ctx); - m_operands.insert(m_operands.begin(), checked_operands.begin(), - checked_operands.end()); - m_return_var = return_var; - m_options = options; - m_wmma_options = wmma_options; - m_wide = false; - m_hi = false; - m_lo = false; - m_uni = false; - m_exit = false; - m_abs = false; - m_neg = false; - m_to_option = false; - m_cache_option = 0; - m_rounding_mode = RN_OPTION; - m_compare_op = -1; - m_saturation_mode = 0; - m_geom_spec = 0; - m_vector_spec = 0; - m_atomic_spec = 0; - m_membar_level = 0; - m_inst_size = 8; // bytes - int rr = 0; - std::list::const_iterator i; - unsigned n = 1; - for (i = wmma_options.begin(); i != wmma_options.end(); i++, n++) { - int last_ptx_inst_option = *i; - switch (last_ptx_inst_option) { - case SYNC_OPTION: - case LOAD_A: - case LOAD_B: - case LOAD_C: - case STORE_D: - case MMA: - m_wmma_type = last_ptx_inst_option; - break; - case ROW: - case COL: - m_wmma_layout[rr++] = last_ptx_inst_option; - break; - case M16N16K16: - case M32N8K16: - case M8N32K16: - break; - default: - assert(0); - break; + } else { + assert( scalar_type.size() < 2 ); + if( scalar_type.size() == 1 ) { + std::list result; + int inst_type = scalar_type.front(); + std::list::const_iterator o; + for( o = operands.begin(); o != operands.end(); o++ ) { + const operand_info &op = *o; + if( op.is_literal() ) { + if( (op.get_type() == double_op_t) && (inst_type == F32_TYPE) ) { + ptx_reg_t v = op.get_literal_value(); + float u = (float)v.f64; + operand_info n(u, ctx); + result.push_back(n); + } else { + result.push_back(op); + } + } else { + result.push_back(op); + } + } + return result; + } } - } - rr = 0; - n = 1; - for (i = options.begin(); i != options.end(); i++, n++) { - int last_ptx_inst_option = *i; - switch (last_ptx_inst_option) { + return operands; +} + + +ptx_instruction::ptx_instruction( int opcode, + const symbol *pred, + int neg_pred, + int pred_mod, + symbol *label, + const std::list &operands, + const operand_info &return_var, + const std::list &options, + const std::list &wmma_options, + const std::list &scalar_type, + memory_space_t space_spec, + const char *file, + unsigned line, + const char *source, + const core_config *config, + gpgpu_context* ctx ) : warp_inst_t(config), m_return_var(ctx) +{ + gpgpu_ctx = ctx; + m_uid = ++(ctx->g_num_ptx_inst_uid); + m_PC = 0; + m_opcode = opcode; + m_pred = pred; + m_neg_pred = neg_pred; + m_pred_mod = pred_mod; + m_label = label; + const std::list checked_operands = check_operands(opcode,scalar_type,operands, ctx); + m_operands.insert(m_operands.begin(), checked_operands.begin(), checked_operands.end() ); + m_return_var = return_var; + m_options = options; + m_wmma_options = wmma_options; + m_wide = false; + m_hi = false; + m_lo = false; + m_uni = false; + m_exit = false; + m_abs = false; + m_neg = false; + m_to_option = false; + m_cache_option = 0; + m_rounding_mode = RN_OPTION; + m_compare_op = -1; + m_saturation_mode = 0; + m_geom_spec = 0; + m_vector_spec = 0; + m_atomic_spec = 0; + m_membar_level = 0; + m_inst_size = 8; // bytes + int rr=0; + std::list::const_iterator i; + unsigned n=1; + for ( i=wmma_options.begin(); i!= wmma_options.end(); i++, n++ ) { + int last_ptx_inst_option = *i; + switch ( last_ptx_inst_option ) { + case SYNC_OPTION: + case LOAD_A: + case LOAD_B: + case LOAD_C: + case STORE_D: + case MMA: + m_wmma_type=last_ptx_inst_option; + break; + case ROW: + case COL: + m_wmma_layout[rr++]=last_ptx_inst_option; + break; + case M16N16K16: + case M32N8K16: + case M8N32K16: + break; + default: + assert(0); + break; + } + } + rr=0; + n=1; + for ( i=options.begin(); i!= options.end(); i++, n++ ) { + int last_ptx_inst_option = *i; + switch ( last_ptx_inst_option ) { case SYNC_OPTION: case ARRIVE_OPTION: case RED_OPTION: - m_barrier_op = last_ptx_inst_option; - break; + m_barrier_op = last_ptx_inst_option; + break; case EQU_OPTION: case NEU_OPTION: case LTU_OPTION: @@ -1280,16 +1186,16 @@ ptx_instruction::ptx_instruction( case GE_OPTION: case LS_OPTION: case HS_OPTION: - m_compare_op = last_ptx_inst_option; - break; + m_compare_op = last_ptx_inst_option; + break; case NUM_OPTION: case NAN_OPTION: - m_compare_op = last_ptx_inst_option; + m_compare_op = last_ptx_inst_option; // assert(0); // finish this - break; + break; case SAT_OPTION: - m_saturation_mode = 1; - break; + m_saturation_mode = 1; + break; case RNI_OPTION: case RZI_OPTION: case RMI_OPTION: @@ -1298,39 +1204,38 @@ ptx_instruction::ptx_instruction( case RZ_OPTION: case RM_OPTION: case RP_OPTION: - m_rounding_mode = last_ptx_inst_option; - break; + m_rounding_mode = last_ptx_inst_option; + break; case HI_OPTION: - m_compare_op = last_ptx_inst_option; - m_hi = true; - assert(!m_lo); - assert(!m_wide); - break; + m_compare_op = last_ptx_inst_option; + m_hi = true; + assert( !m_lo ); + assert( !m_wide ); + break; case LO_OPTION: - m_compare_op = last_ptx_inst_option; - m_lo = true; - assert(!m_hi); - assert(!m_wide); - break; + m_compare_op = last_ptx_inst_option; + m_lo = true; + assert( !m_hi ); + assert( !m_wide ); + break; case WIDE_OPTION: - m_wide = true; - assert(!m_lo); - assert(!m_hi); - break; + m_wide = true; + assert( !m_lo ); + assert( !m_hi ); + break; case UNI_OPTION: - m_uni = true; // don't care... < now we DO care when constructing - // flowgraph> - break; + m_uni = true; // don't care... < now we DO care when constructing flowgraph> + break; case GEOM_MODIFIER_1D: case GEOM_MODIFIER_2D: case GEOM_MODIFIER_3D: - m_geom_spec = last_ptx_inst_option; - break; + m_geom_spec = last_ptx_inst_option; + break; case V2_TYPE: case V3_TYPE: case V4_TYPE: - m_vector_spec = last_ptx_inst_option; - break; + m_vector_spec = last_ptx_inst_option; + break; case ATOMIC_AND: case ATOMIC_OR: case ATOMIC_XOR: @@ -1341,225 +1246,223 @@ ptx_instruction::ptx_instruction( case ATOMIC_DEC: case ATOMIC_MIN: case ATOMIC_MAX: - m_atomic_spec = last_ptx_inst_option; - break; + m_atomic_spec = last_ptx_inst_option; + break; case APPROX_OPTION: - break; + break; case FULL_OPTION: - break; + break; case ANY_OPTION: - m_vote_mode = vote_any; - break; + m_vote_mode = vote_any; + break; case ALL_OPTION: - m_vote_mode = vote_all; - break; + m_vote_mode = vote_all; + break; case BALLOT_OPTION: - m_vote_mode = vote_ballot; - break; + m_vote_mode = vote_ballot; + break; case GLOBAL_OPTION: - m_membar_level = GLOBAL_OPTION; - break; + m_membar_level = GLOBAL_OPTION; + break; case CTA_OPTION: - m_membar_level = CTA_OPTION; - break; + m_membar_level = CTA_OPTION; + break; case SYS_OPTION: - m_membar_level = SYS_OPTION; - break; + m_membar_level = SYS_OPTION; + break; case FTZ_OPTION: - break; + break; case EXIT_OPTION: - m_exit = true; - break; + m_exit = true; + break; case ABS_OPTION: - m_abs = true; - break; + m_abs = true; + break; case NEG_OPTION: - m_neg = true; - break; + m_neg = true; + break; case TO_OPTION: - m_to_option = true; - break; - case CA_OPTION: - case CG_OPTION: - case CS_OPTION: - case LU_OPTION: - case CV_OPTION: - m_cache_option = last_ptx_inst_option; - break; + m_to_option = true; + break; + case CA_OPTION: case CG_OPTION: case CS_OPTION: case LU_OPTION: case CV_OPTION: + m_cache_option = last_ptx_inst_option; + break; case HALF_OPTION: - m_inst_size = 4; // bytes - break; + m_inst_size = 4; // bytes + break; case EXTP_OPTION: - break; + break; case NC_OPTION: - m_cache_option = last_ptx_inst_option; - break; + m_cache_option = last_ptx_inst_option; + break; case UP_OPTION: case DOWN_OPTION: case BFLY_OPTION: case IDX_OPTION: - m_shfl_op = last_ptx_inst_option; - break; + m_shfl_op = last_ptx_inst_option; + break; case PRMT_F4E_MODE: case PRMT_B4E_MODE: case PRMT_RC8_MODE: case PRMT_ECL_MODE: case PRMT_ECR_MODE: case PRMT_RC16_MODE: - m_prmt_op = last_ptx_inst_option; - break; + m_prmt_op = last_ptx_inst_option; + break; default: - assert(0); - break; - } - } - m_scalar_type = scalar_type; - m_space_spec = space_spec; - if ((opcode == ST_OP || opcode == LD_OP || opcode == LDU_OP) && - (space_spec == undefined_space)) { - m_space_spec = generic_space; - } - for (std::vector::const_iterator i = m_operands.begin(); - i != m_operands.end(); ++i) { - const operand_info &op = *i; - if (op.get_addr_space() != undefined_space) - m_space_spec = op.get_addr_space(); // TODO: can have more than one - // memory space for ptxplus (g8x) - // inst - } - if (opcode == TEX_OP) m_space_spec = tex_space; - - m_source_file = file ? file : ""; - m_source_line = line; - m_source = source; - // Trim tabs - m_source.erase(std::remove(m_source.begin(), m_source.end(), '\t'), - m_source.end()); - - if (opcode == CALL_OP) { - const operand_info &target = func_addr(); - assert(target.is_function_address()); - const symbol *func_addr = target.get_symbol(); - const function_info *target_func = func_addr->get_pc(); - std::string fname = target_func->get_name(); - - if (fname == "vprintf") { - m_is_printf = true; - } - if (fname == "cudaStreamCreateWithFlags") m_is_cdp = 1; - if (fname == "cudaGetParameterBufferV2") m_is_cdp = 2; - if (fname == "cudaLaunchDeviceV2") m_is_cdp = 4; - } -} - -void ptx_instruction::print_insn() const { - print_insn(stdout); - fflush(stdout); -} - -void ptx_instruction::print_insn(FILE *fp) const { - fprintf(fp, "%s", to_string().c_str()); -} - -std::string ptx_instruction::to_string() const { - char buf[STR_SIZE]; - unsigned used_bytes = 0; - if (!is_label()) { - used_bytes += - snprintf(buf + used_bytes, STR_SIZE - used_bytes, " PC=0x%03x ", m_PC); - } else { - used_bytes += - snprintf(buf + used_bytes, STR_SIZE - used_bytes, " "); - } - used_bytes += - snprintf(buf + used_bytes, STR_SIZE - used_bytes, "(%s:%d) %s", - m_source_file.c_str(), m_source_line, m_source.c_str()); - return std::string(buf); -} -operand_info ptx_instruction::get_pred() const { - return operand_info(m_pred, gpgpu_ctx); -} - -function_info::function_info(int entry_point, gpgpu_context *ctx) { - gpgpu_ctx = ctx; - m_uid = (gpgpu_ctx->function_info_sm_next_uid)++; - m_entry_point = (entry_point == 1) ? true : false; - m_extern = (entry_point == 2) ? true : false; - num_reconvergence_pairs = 0; - m_symtab = NULL; - m_assembled = false; - m_return_var_sym = NULL; - m_kernel_info.cmem = 0; - m_kernel_info.lmem = 0; - m_kernel_info.regs = 0; - m_kernel_info.smem = 0; - m_local_mem_framesize = 0; - m_args_aligned_size = -1; - pdom_done = false; // initialize it to false -} - -unsigned function_info::print_insn(unsigned pc, FILE *fp) const { - unsigned inst_size = 1; // return offset to next instruction or 1 if unknown - unsigned index = pc - m_start_PC; - char command[1024]; - char buffer[1024]; - memset(command, 0, 1024); - memset(buffer, 0, 1024); - snprintf(command, 1024, "c++filt -p %s", m_name.c_str()); - FILE *p = popen(command, "r"); - buffer[0] = 0; - assert(fgets(buffer, 1023, p) != NULL); - // Remove trailing "\n" in buffer - char *c; - if ((c = strchr(buffer, '\n')) != NULL) *c = '\0'; - fprintf(fp, "%s", buffer); - if (index >= m_instr_mem_size) { - fprintf(fp, "", - m_start_PC + m_instr_mem_size - 1); - } else { - if (m_instr_mem[index] != NULL) { - m_instr_mem[index]->print_insn(fp); - inst_size = m_instr_mem[index]->isize; - } else - fprintf(fp, "", pc); - } - pclose(p); - return inst_size; -} - -std::string function_info::get_insn_str(unsigned pc) const { - unsigned index = pc - m_start_PC; - if (index >= m_instr_mem_size) { - char buff[STR_SIZE]; - buff[STR_SIZE - 1] = '\0'; - snprintf(buff, STR_SIZE, "", - m_start_PC + m_instr_mem_size - 1); - return std::string(buff); - } else { - if (m_instr_mem[index] != NULL) { - return m_instr_mem[index]->to_string(); - } else { + assert(0); + break; + } + } + m_scalar_type = scalar_type; + m_space_spec = space_spec; + if( ( opcode == ST_OP || opcode == LD_OP || opcode == LDU_OP ) && (space_spec == undefined_space) ) { + m_space_spec = generic_space; + } + for( std::vector::const_iterator i=m_operands.begin(); i!=m_operands.end(); ++i) { + const operand_info &op = *i; + if( op.get_addr_space() != undefined_space ) + m_space_spec = op.get_addr_space(); // TODO: can have more than one memory space for ptxplus (g8x) inst + } + if( opcode == TEX_OP ) + m_space_spec = tex_space; + + m_source_file = file?file:""; + m_source_line = line; + m_source = source; + // Trim tabs + m_source.erase( std::remove( m_source.begin(), m_source.end(), '\t' ), m_source.end() ); + + if (opcode == CALL_OP) { + const operand_info &target = func_addr(); + assert( target.is_function_address() ); + const symbol *func_addr = target.get_symbol(); + const function_info *target_func = func_addr->get_pc(); + std::string fname = target_func->get_name(); + + if (fname =="vprintf"){ + m_is_printf = true; + } + if(fname == "cudaStreamCreateWithFlags") + m_is_cdp = 1; + if(fname == "cudaGetParameterBufferV2") + m_is_cdp = 2; + if(fname == "cudaLaunchDeviceV2") + m_is_cdp = 4; + + } +} + +void ptx_instruction::print_insn() const +{ + print_insn(stdout); + fflush(stdout); +} + +void ptx_instruction::print_insn( FILE *fp ) const +{ + fprintf( fp, "%s", to_string().c_str() ); +} + +std::string ptx_instruction::to_string() const +{ + char buf[ STR_SIZE ]; + unsigned used_bytes = 0; + if( !is_label() ) { + used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, " PC=0x%03x ", m_PC ); + } else { + used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, " " ); + } + used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, + "(%s:%d) %s", + m_source_file.c_str(), m_source_line, + m_source.c_str() ); + return std::string( buf ); +} +operand_info ptx_instruction::get_pred() const +{ + return operand_info( m_pred, gpgpu_ctx); +} + + +function_info::function_info(int entry_point, gpgpu_context* ctx ) +{ + gpgpu_ctx = ctx; + m_uid = (gpgpu_ctx->function_info_sm_next_uid)++; + m_entry_point = (entry_point==1)?true:false; + m_extern = (entry_point==2)?true:false; + num_reconvergence_pairs = 0; + m_symtab = NULL; + m_assembled = false; + m_return_var_sym = NULL; + m_kernel_info.cmem = 0; + m_kernel_info.lmem = 0; + m_kernel_info.regs = 0; + m_kernel_info.smem = 0; + m_local_mem_framesize = 0; + m_args_aligned_size = -1; + pdom_done = false; //initialize it to false +} + +unsigned function_info::print_insn( unsigned pc, FILE * fp ) const +{ + unsigned inst_size=1; // return offset to next instruction or 1 if unknown + unsigned index = pc - m_start_PC; + char command[1024]; + char buffer[1024]; + memset(command, 0, 1024); + memset(buffer, 0, 1024); + snprintf(command,1024,"c++filt -p %s",m_name.c_str()); + FILE *p = popen(command,"r"); + buffer[0]=0; + assert(fgets(buffer, 1023, p) != NULL); + // Remove trailing "\n" in buffer + char *c; + if ((c=strchr(buffer, '\n')) != NULL) *c = '\0'; + fprintf(fp,"%s",buffer); + if ( index >= m_instr_mem_size ) { + fprintf(fp, "", m_start_PC + m_instr_mem_size - 1 ); + } else { + if ( m_instr_mem[index] != NULL ) { + m_instr_mem[index]->print_insn(fp); + inst_size = m_instr_mem[index]->isize; + } else + fprintf(fp, "", pc ); + } + pclose(p); + return inst_size; +} + +std::string function_info::get_insn_str( unsigned pc ) const +{ + unsigned index = pc - m_start_PC; + if ( index >= m_instr_mem_size ) { char buff[STR_SIZE]; - buff[STR_SIZE - 1] = '\0'; - snprintf(buff, STR_SIZE, "", pc); + buff[STR_SIZE-1] = '\0'; + snprintf(buff, STR_SIZE, "", m_start_PC + m_instr_mem_size - 1 ); return std::string(buff); + } else { + if ( m_instr_mem[index] != NULL ) { + return m_instr_mem[index]->to_string(); + } else { + char buff[STR_SIZE]; + buff[STR_SIZE-1] = '\0'; + snprintf(buff, STR_SIZE, "", pc ); + return std::string(buff); + } + } +} + +void gpgpu_ptx_assemble( std::string kname, void *kinfo ) +{ + function_info *func_info = (function_info *)kinfo; + if((function_info *)kinfo == NULL) { + printf("GPGPU-Sim PTX: Warning - missing function definition \'%s\'\n", kname.c_str()); + return; + } + if( func_info->is_extern() ) { + printf("GPGPU-Sim PTX: skipping assembly for extern declared function \'%s\'\n", func_info->get_name().c_str() ); + return; } - } -} - -void gpgpu_ptx_assemble(std::string kname, void *kinfo) { - function_info *func_info = (function_info *)kinfo; - if ((function_info *)kinfo == NULL) { - printf("GPGPU-Sim PTX: Warning - missing function definition \'%s\'\n", - kname.c_str()); - return; - } - if (func_info->is_extern()) { - printf( - "GPGPU-Sim PTX: skipping assembly for extern declared function " - "\'%s\'\n", - func_info->get_name().c_str()); - return; - } - func_info->ptx_assemble(); + func_info->ptx_assemble(); } diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 6d82374..f4c5c37 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -32,13 +30,13 @@ #include "../abstract_hardware_model.h" -#include #include #include +#include #include #include -#include #include +#include //#include "ptx.tab.h" #include "ptx_sim.h" @@ -48,1518 +46,1537 @@ class gpgpu_context; class type_info_key { - public: - type_info_key() { - m_is_non_arch_reg = false; - m_init = false; - } - type_info_key(memory_space_t space_spec, int scalar_type_spec, - int vector_spec, int alignment_spec, int extern_spec, - int array_dim) { - m_is_non_arch_reg = false; - m_init = true; - m_space_spec = space_spec; - m_scalar_type_spec = scalar_type_spec; - m_vector_spec = vector_spec; - m_alignment_spec = alignment_spec; - m_extern_spec = extern_spec; - m_array_dim = array_dim; - m_is_function = 0; - } - void set_is_func() { - assert(!m_init); - m_init = true; - m_space_spec = undefined_space; - m_scalar_type_spec = 0; - m_vector_spec = 0; - m_alignment_spec = 0; - m_extern_spec = 0; - m_array_dim = 0; - m_is_function = 1; - } - - void set_array_dim(int array_dim) { m_array_dim = array_dim; } - int get_array_dim() const { - assert(m_init); - return m_array_dim; - } - void set_is_non_arch_reg() { m_is_non_arch_reg = true; } - - bool is_non_arch_reg() const { return m_is_non_arch_reg; } - bool is_reg() const { return m_space_spec == reg_space; } - bool is_param_kernel() const { return m_space_spec == param_space_kernel; } - bool is_param_local() const { return m_space_spec == param_space_local; } - bool is_param_unclassified() const { - return m_space_spec == param_space_unclassified; - } - bool is_global() const { return m_space_spec == global_space; } - bool is_local() const { return m_space_spec == local_space; } - bool is_shared() const { return m_space_spec == shared_space; } - bool is_const() const { return m_space_spec.get_type() == const_space; } - bool is_tex() const { return m_space_spec == tex_space; } - bool is_func_addr() const { return m_is_function ? true : false; } - int scalar_type() const { return m_scalar_type_spec; } - int get_alignment_spec() const { return m_alignment_spec; } - unsigned type_decode(size_t &size, int &t) const; - static unsigned type_decode(int type, size_t &size, int &t); - memory_space_t get_memory_space() const { return m_space_spec; } - - private: - bool m_init; - memory_space_t m_space_spec; - int m_scalar_type_spec; - int m_vector_spec; - int m_alignment_spec; - int m_extern_spec; - int m_array_dim; - int m_is_function; - bool m_is_non_arch_reg; - - friend struct type_info_key_compare; +public: + type_info_key() + { + m_is_non_arch_reg = false; + m_init = false; + } + type_info_key( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec, int array_dim ) + { + m_is_non_arch_reg = false; + m_init = true; + m_space_spec = space_spec; + m_scalar_type_spec = scalar_type_spec; + m_vector_spec = vector_spec; + m_alignment_spec = alignment_spec; + m_extern_spec = extern_spec; + m_array_dim = array_dim; + m_is_function = 0; + } + void set_is_func() + { + assert(!m_init); + m_init = true; + m_space_spec = undefined_space; + m_scalar_type_spec = 0; + m_vector_spec = 0; + m_alignment_spec = 0; + m_extern_spec = 0; + m_array_dim = 0; + m_is_function = 1; + } + + void set_array_dim( int array_dim ) { m_array_dim = array_dim; } + int get_array_dim() const { assert(m_init); return m_array_dim; } + void set_is_non_arch_reg() { m_is_non_arch_reg = true; } + + bool is_non_arch_reg() const { return m_is_non_arch_reg; } + bool is_reg() const { return m_space_spec == reg_space;} + bool is_param_kernel() const { return m_space_spec == param_space_kernel;} + bool is_param_local() const { return m_space_spec == param_space_local; } + bool is_param_unclassified() const { return m_space_spec == param_space_unclassified; } + bool is_global() const { return m_space_spec == global_space;} + bool is_local() const { return m_space_spec == local_space;} + bool is_shared() const { return m_space_spec == shared_space;} + bool is_const() const { return m_space_spec.get_type() == const_space;} + bool is_tex() const { return m_space_spec == tex_space;} + bool is_func_addr() const { return m_is_function?true:false; } + int scalar_type() const { return m_scalar_type_spec;} + int get_alignment_spec() const { return m_alignment_spec;} + unsigned type_decode( size_t &size, int &t ) const; + static unsigned type_decode( int type, size_t &size, int &t ); + memory_space_t get_memory_space() const { return m_space_spec; } +private: + bool m_init; + memory_space_t m_space_spec; + int m_scalar_type_spec; + int m_vector_spec; + int m_alignment_spec; + int m_extern_spec; + int m_array_dim; + int m_is_function; + bool m_is_non_arch_reg; + + friend struct type_info_key_compare; }; class symbol_table; struct type_info_key_compare { - bool operator()(const type_info_key &a, const type_info_key &b) const { - assert(a.m_init && b.m_init); - if (a.m_space_spec < b.m_space_spec) return true; - if (a.m_scalar_type_spec < b.m_scalar_type_spec) return true; - if (a.m_vector_spec < b.m_vector_spec) return true; - if (a.m_alignment_spec < b.m_alignment_spec) return true; - if (a.m_extern_spec < b.m_extern_spec) return true; - if (a.m_array_dim < b.m_array_dim) return true; - if (a.m_is_function < b.m_is_function) return true; - - return false; - } + bool operator()( const type_info_key &a, const type_info_key &b ) const + { + assert( a.m_init && b.m_init ); + if ( a.m_space_spec < b.m_space_spec ) return true; + if ( a.m_scalar_type_spec < b.m_scalar_type_spec ) return true; + if ( a.m_vector_spec < b.m_vector_spec ) return true; + if ( a.m_alignment_spec < b.m_alignment_spec ) return true; + if ( a.m_extern_spec < b.m_extern_spec ) return true; + if ( a.m_array_dim < b.m_array_dim ) return true; + if ( a.m_is_function < b.m_is_function ) return true; + + return false; + } }; class type_info { - public: - type_info(symbol_table *scope, type_info_key t) { m_type_info = t; } - const type_info_key &get_key() const { return m_type_info; } - - private: - symbol_table *m_scope; - type_info_key m_type_info; +public: + type_info( symbol_table *scope, type_info_key t ) + { + m_type_info = t; + } + const type_info_key &get_key() const { return m_type_info;} + +private: + symbol_table *m_scope; + type_info_key m_type_info; }; enum operand_type { - reg_t, - vector_t, - builtin_t, - address_t, - memory_t, - float_op_t, - double_op_t, - int_t, - unsigned_t, - symbolic_t, - label_t, - v_reg_t, - v_float_op_t, - v_double_op_t, - v_int_t, - v_unsigned_t, - undef_t + reg_t, vector_t, builtin_t, address_t, memory_t, float_op_t, double_op_t, int_t, + unsigned_t, symbolic_t, label_t, v_reg_t, v_float_op_t, v_double_op_t, + v_int_t, v_unsigned_t, undef_t }; class operand_info; class symbol { - public: - symbol(const char *name, const type_info *type, const char *location, - unsigned size, gpgpu_context *ctx) { - gpgpu_ctx = ctx; - m_uid = get_uid(); - m_name = name; - m_decl_location = location; - m_type = type; - m_size = size; - m_address_valid = false; - m_is_label = false; - m_is_shared = false; - m_is_const = false; - m_is_global = false; - m_is_local = false; - m_is_param_local = false; - m_is_param_kernel = false; - m_is_tex = false; - m_is_func_addr = false; - m_reg_num_valid = false; - m_function = NULL; - m_reg_num = (unsigned)-1; - m_arch_reg_num = (unsigned)-1; - m_address = (unsigned)-1; - m_initializer.clear(); - if (type) m_is_shared = type->get_key().is_shared(); - if (type) m_is_const = type->get_key().is_const(); - if (type) m_is_global = type->get_key().is_global(); - if (type) m_is_local = type->get_key().is_local(); - if (type) m_is_param_local = type->get_key().is_param_local(); - if (type) m_is_param_kernel = type->get_key().is_param_kernel(); - if (type) m_is_tex = type->get_key().is_tex(); - if (type) m_is_func_addr = type->get_key().is_func_addr(); - } - unsigned get_size_in_bytes() const { return m_size; } - const std::string &name() const { return m_name; } - const std::string &decl_location() const { return m_decl_location; } - const type_info *type() const { return m_type; } - addr_t get_address() const { - assert(m_is_label || - !m_type->get_key().is_reg()); // todo : other assertions - assert(m_address_valid); - return m_address; - } - function_info *get_pc() const { return m_function; } - void set_regno(unsigned regno, unsigned arch_regno) { - m_reg_num_valid = true; - m_reg_num = regno; - m_arch_reg_num = arch_regno; - } - - void set_address(addr_t addr) { - m_address_valid = true; - m_address = addr; - } - void set_label_address(addr_t addr) { - m_address_valid = true; - m_address = addr; - m_is_label = true; - } - void set_function(function_info *func) { - m_function = func; - m_is_func_addr = true; - } - - bool is_label() const { return m_is_label; } - bool is_shared() const { return m_is_shared; } - bool is_sstarr() const { return m_is_sstarr; } - bool is_const() const { return m_is_const; } - bool is_global() const { return m_is_global; } - bool is_local() const { return m_is_local; } - bool is_param_local() const { return m_is_param_local; } - bool is_param_kernel() const { return m_is_param_kernel; } - bool is_tex() const { return m_is_tex; } - bool is_func_addr() const { return m_is_func_addr; } - bool is_reg() const { - if (m_type == NULL) { - return false; - } - return m_type->get_key().is_reg(); - } - bool is_non_arch_reg() const { - if (m_type == NULL) { - return false; - } - return m_type->get_key().is_non_arch_reg(); - } - - void add_initializer(const std::list &init); - bool has_initializer() const { return m_initializer.size() > 0; } - std::list get_initializer() const { return m_initializer; } - unsigned reg_num() const { - assert(m_reg_num_valid); - return m_reg_num; - } - unsigned arch_reg_num() const { - assert(m_reg_num_valid); - return m_arch_reg_num; - } - void print_info(FILE *fp) const; - unsigned uid() const { return m_uid; } - - private: - gpgpu_context *gpgpu_ctx; - unsigned get_uid(); - unsigned m_uid; - const type_info *m_type; - unsigned m_size; // in bytes - std::string m_name; - std::string m_decl_location; - - unsigned m_address; - function_info *m_function; // used for function symbols - - bool m_address_valid; - bool m_is_label; - bool m_is_shared; - bool m_is_sstarr; - bool m_is_const; - bool m_is_global; - bool m_is_local; - bool m_is_param_local; - bool m_is_param_kernel; - bool m_is_tex; - bool m_is_func_addr; - unsigned m_reg_num; - unsigned m_arch_reg_num; - bool m_reg_num_valid; - - std::list m_initializer; +public: + symbol( const char *name, const type_info *type, const char *location, unsigned size, gpgpu_context* ctx ) + { + gpgpu_ctx = ctx; + m_uid = get_uid(); + m_name = name; + m_decl_location = location; + m_type = type; + m_size = size; + m_address_valid = false; + m_is_label = false; + m_is_shared = false; + m_is_const = false; + m_is_global = false; + m_is_local = false; + m_is_param_local = false; + m_is_param_kernel = false; + m_is_tex = false; + m_is_func_addr = false; + m_reg_num_valid = false; + m_function = NULL; + m_reg_num=(unsigned)-1; + m_arch_reg_num=(unsigned)-1; + m_address=(unsigned)-1; + m_initializer.clear(); + if ( type ) m_is_shared = type->get_key().is_shared(); + if ( type ) m_is_const = type->get_key().is_const(); + if ( type ) m_is_global = type->get_key().is_global(); + if ( type ) m_is_local = type->get_key().is_local(); + if ( type ) m_is_param_local = type->get_key().is_param_local(); + if ( type ) m_is_param_kernel = type->get_key().is_param_kernel(); + if ( type ) m_is_tex = type->get_key().is_tex(); + if ( type ) m_is_func_addr = type->get_key().is_func_addr(); + } + unsigned get_size_in_bytes() const + { + return m_size; + } + const std::string &name() const { return m_name;} + const std::string &decl_location() const { return m_decl_location;} + const type_info *type() const { return m_type;} + addr_t get_address() const + { + assert( m_is_label || !m_type->get_key().is_reg() ); // todo : other assertions + assert( m_address_valid ); + return m_address; + } + function_info *get_pc() const + { + return m_function; + } + void set_regno( unsigned regno, unsigned arch_regno ) + { + m_reg_num_valid = true; + m_reg_num = regno; + m_arch_reg_num = arch_regno; + } + + void set_address( addr_t addr ) + { + m_address_valid = true; + m_address = addr; + } + void set_label_address( addr_t addr) + { + m_address_valid = true; + m_address = addr; + m_is_label = true; + } + void set_function( function_info *func ) + { + m_function = func; + m_is_func_addr = true; + } + + bool is_label() const { return m_is_label;} + bool is_shared() const { return m_is_shared;} + bool is_sstarr() const { return m_is_sstarr;} + bool is_const() const { return m_is_const;} + bool is_global() const { return m_is_global;} + bool is_local() const { return m_is_local;} + bool is_param_local() const { return m_is_param_local; } + bool is_param_kernel() const { return m_is_param_kernel; } + bool is_tex() const { return m_is_tex;} + bool is_func_addr() const { return m_is_func_addr; } + bool is_reg() const + { + if ( m_type == NULL ) { + return false; + } + return m_type->get_key().is_reg(); + } + bool is_non_arch_reg() const + { + if ( m_type == NULL ) { + return false; + } + return m_type->get_key().is_non_arch_reg(); + } + + void add_initializer( const std::list &init ); + bool has_initializer() const + { + return m_initializer.size() > 0; + } + std::list get_initializer() const + { + return m_initializer; + } + unsigned reg_num() const + { + assert( m_reg_num_valid ); + return m_reg_num; + } + unsigned arch_reg_num() const + { + assert( m_reg_num_valid ); + return m_arch_reg_num; + } + void print_info(FILE *fp) const; + unsigned uid() const { return m_uid; } + +private: + gpgpu_context* gpgpu_ctx; + unsigned get_uid(); + unsigned m_uid; + const type_info *m_type; + unsigned m_size; // in bytes + std::string m_name; + std::string m_decl_location; + + unsigned m_address; + function_info *m_function; // used for function symbols + + bool m_address_valid; + bool m_is_label; + bool m_is_shared; + bool m_is_sstarr; + bool m_is_const; + bool m_is_global; + bool m_is_local; + bool m_is_param_local; + bool m_is_param_kernel; + bool m_is_tex; + bool m_is_func_addr; + unsigned m_reg_num; + unsigned m_arch_reg_num; + bool m_reg_num_valid; + + std::list m_initializer; }; class symbol_table { - public: - symbol_table(); - symbol_table(const char *scope_name, unsigned entry_point, - symbol_table *parent, gpgpu_context *ctx); - void set_name(const char *name); - const ptx_version &get_ptx_version() const; - unsigned get_sm_target() const; - void set_ptx_version(float ver, unsigned ext); - void set_sm_target(const char *target, const char *ext, const char *ext2); - symbol *lookup(const char *identifier); - std::string get_scope_name() const { return m_scope_name; } - symbol *add_variable(const char *identifier, const type_info *type, - unsigned size, const char *filename, unsigned line); - void add_function(function_info *func, const char *filename, - unsigned linenumber); - bool add_function_decl(const char *name, int entry_point, - function_info **func_info, - symbol_table **symbol_table); - function_info *lookup_function(std::string name); - type_info *add_type(memory_space_t space_spec, int scalar_type_spec, - int vector_spec, int alignment_spec, int extern_spec); - type_info *add_type(function_info *func); - type_info *get_array_type(type_info *base_type, unsigned array_dim); - void set_label_address(const symbol *label, unsigned addr); - unsigned next_reg_num() { return ++m_reg_allocator; } - addr_t get_shared_next() { return m_shared_next; } - addr_t get_sstarr_next() { return m_sstarr_next; } - addr_t get_global_next() { return m_global_next; } - addr_t get_local_next() { return m_local_next; } - addr_t get_tex_next() { return m_tex_next; } - void alloc_shared(unsigned num_bytes) { m_shared_next += num_bytes; } - void alloc_sstarr(unsigned num_bytes) { m_sstarr_next += num_bytes; } - void alloc_global(unsigned num_bytes) { m_global_next += num_bytes; } - void alloc_local(unsigned num_bytes) { m_local_next += num_bytes; } - void alloc_tex(unsigned num_bytes) { m_tex_next += num_bytes; } - - typedef std::list::iterator iterator; - - iterator global_iterator_begin() { return m_globals.begin(); } - iterator global_iterator_end() { return m_globals.end(); } - - iterator const_iterator_begin() { return m_consts.begin(); } - iterator const_iterator_end() { return m_consts.end(); } - - void dump(); - - // Jin: handle instruction group for cdp - symbol_table *start_inst_group(); - symbol_table *end_inst_group(); - - // backward pointer - class gpgpu_context *gpgpu_ctx; - - private: - unsigned m_reg_allocator; - unsigned m_shared_next; - unsigned m_sstarr_next; - unsigned m_const_next; - unsigned m_global_next; - unsigned m_local_next; - unsigned m_tex_next; - - symbol_table *m_parent; - ptx_version m_ptx_version; - std::string m_scope_name; - std::map - m_symbols; // map from name of register to pointers to the registers - std::map m_types; - std::list m_globals; - std::list m_consts; - std::map m_function_info_lookup; - std::map m_function_symtab_lookup; - - // Jin: handle instruction group for cdp - unsigned m_inst_group_id; - std::map m_inst_group_symtab; +public: + symbol_table(); + symbol_table( const char *scope_name, unsigned entry_point, symbol_table *parent, gpgpu_context* ctx); + void set_name( const char *name ); + const ptx_version &get_ptx_version() const; + unsigned get_sm_target() const; + void set_ptx_version( float ver, unsigned ext ); + void set_sm_target( const char *target, const char *ext, const char *ext2 ); + symbol* lookup( const char *identifier ); + std::string get_scope_name() const { return m_scope_name; } + symbol *add_variable( const char *identifier, const type_info *type, unsigned size, const char *filename, unsigned line ); + void add_function( function_info *func, const char *filename, unsigned linenumber ); + bool add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **symbol_table ); + function_info *lookup_function(std::string name); + type_info *add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ); + type_info *add_type( function_info *func ); + type_info *get_array_type( type_info *base_type, unsigned array_dim ); + void set_label_address( const symbol *label, unsigned addr ); + unsigned next_reg_num() { return ++m_reg_allocator;} + addr_t get_shared_next() { return m_shared_next;} + addr_t get_sstarr_next() { return m_sstarr_next;} + addr_t get_global_next() { return m_global_next;} + addr_t get_local_next() { return m_local_next;} + addr_t get_tex_next() { return m_tex_next;} + void alloc_shared( unsigned num_bytes ) { m_shared_next += num_bytes;} + void alloc_sstarr( unsigned num_bytes ) { m_sstarr_next += num_bytes;} + void alloc_global( unsigned num_bytes ) { m_global_next += num_bytes;} + void alloc_local( unsigned num_bytes ) { m_local_next += num_bytes;} + void alloc_tex( unsigned num_bytes ) { m_tex_next += num_bytes;} + + typedef std::list::iterator iterator; + + iterator global_iterator_begin() { return m_globals.begin();} + iterator global_iterator_end() { return m_globals.end();} + + iterator const_iterator_begin() { return m_consts.begin();} + iterator const_iterator_end() { return m_consts.end();} + + void dump(); + + //Jin: handle instruction group for cdp + symbol_table* start_inst_group(); + symbol_table* end_inst_group(); + + // backward pointer + class gpgpu_context* gpgpu_ctx; + +private: + unsigned m_reg_allocator; + unsigned m_shared_next; + unsigned m_sstarr_next; + unsigned m_const_next; + unsigned m_global_next; + unsigned m_local_next; + unsigned m_tex_next; + + symbol_table *m_parent; + ptx_version m_ptx_version; + std::string m_scope_name; + std::map m_symbols; //map from name of register to pointers to the registers + std::map m_types; + std::list m_globals; + std::list m_consts; + std::map m_function_info_lookup; + std::map m_function_symtab_lookup; + + //Jin: handle instruction group for cdp + unsigned m_inst_group_id; + std::map m_inst_group_symtab; }; class operand_info { - public: - operand_info(gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = false; - m_immediate_address = false; - m_addr_offset = 0; - m_value.m_symbolic = NULL; - } - operand_info(const symbol *addr, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - if (addr->is_label()) { - m_type = label_t; - } else if (addr->is_shared()) { - m_type = symbolic_t; - } else if (addr->is_const()) { - m_type = symbolic_t; - } else if (addr->is_global()) { - m_type = symbolic_t; - } else if (addr->is_local()) { - m_type = symbolic_t; - } else if (addr->is_param_local()) { - m_type = symbolic_t; - } else if (addr->is_param_kernel()) { - m_type = symbolic_t; - } else if (addr->is_tex()) { - m_type = symbolic_t; - } else if (addr->is_func_addr()) { - m_type = symbolic_t; - } else if (!addr->is_reg()) { - m_type = symbolic_t; - } else { - m_type = reg_t; - } - - m_is_non_arch_reg = addr->is_non_arch_reg(); - m_value.m_symbolic = addr; - m_addr_offset = 0; - m_vector = false; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } - operand_info(const symbol *addr1, const symbol *addr2, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_type = memory_t; - m_value.m_vector_symbolic = new const symbol *[8]; - m_value.m_vector_symbolic[0] = addr1; - m_value.m_vector_symbolic[1] = addr2; - m_value.m_vector_symbolic[2] = NULL; - m_value.m_vector_symbolic[3] = NULL; - m_value.m_vector_symbolic[4] = NULL; - m_value.m_vector_symbolic[5] = NULL; - m_value.m_vector_symbolic[6] = NULL; - m_value.m_vector_symbolic[7] = NULL; - m_addr_offset = 0; - m_vector = false; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } - operand_info(int builtin_id, int dim_mod, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = builtin_t; - m_value.m_int = builtin_id; - m_addr_offset = dim_mod; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } - operand_info(const symbol *addr, int offset, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = address_t; - m_value.m_symbolic = addr; - m_addr_offset = offset; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } - operand_info(unsigned x, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = unsigned_t; - m_value.m_unsigned = x; - m_addr_offset = x; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = true; - } - operand_info(int x, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = int_t; - m_value.m_int = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } - operand_info(float x, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = float_op_t; - m_value.m_float = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } - operand_info(double x, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = double_op_t; - m_value.m_double = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } - operand_info(const symbol *s1, const symbol *s2, const symbol *s3, - const symbol *s4, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = true; - m_type = vector_t; - m_value.m_vector_symbolic = new const symbol *[8]; - m_value.m_vector_symbolic[0] = s1; - m_value.m_vector_symbolic[1] = s2; - m_value.m_vector_symbolic[2] = s3; - m_value.m_vector_symbolic[3] = s4; - m_value.m_vector_symbolic[4] = NULL; - m_value.m_vector_symbolic[5] = NULL; - m_value.m_vector_symbolic[6] = NULL; - m_value.m_vector_symbolic[7] = NULL; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } - operand_info(const symbol *s1, const symbol *s2, const symbol *s3, - const symbol *s4, const symbol *s5, const symbol *s6, - const symbol *s7, const symbol *s8, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = true; - m_type = vector_t; - m_value.m_vector_symbolic = new const symbol *[8]; - m_value.m_vector_symbolic[0] = s1; - m_value.m_vector_symbolic[1] = s2; - m_value.m_vector_symbolic[2] = s3; - m_value.m_vector_symbolic[3] = s4; - m_value.m_vector_symbolic[4] = s5; - m_value.m_vector_symbolic[5] = s6; - m_value.m_vector_symbolic[6] = s7; - m_value.m_vector_symbolic[7] = s8; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } - - void init(gpgpu_context *ctx) { - gpgpu_ctx = ctx; - m_uid = (unsigned)-1; - m_valid = false; - m_vector = false; - m_type = undef_t; - m_immediate_address = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = (unsigned)-1; - m_value.m_int = 0; - m_value.m_unsigned = (unsigned)-1; - m_value.m_float = 0; - m_value.m_double = 0; - for (unsigned i = 0; i < 4; i++) { - m_value.m_vint[i] = 0; - m_value.m_vunsigned[i] = 0; - m_value.m_vfloat[i] = 0; - m_value.m_vdouble[i] = 0; - } - m_value.m_symbolic = NULL; - m_value.m_vector_symbolic = NULL; - m_addr_offset = 0; - m_neg_pred = 0; - m_is_return_var = 0; - m_is_non_arch_reg = 0; - } - void make_memory_operand() { m_type = memory_t; } - void set_return() { m_is_return_var = true; } - void set_immediate_addr() { m_immediate_address = true; } - const std::string &name() const { - assert(m_type == symbolic_t || m_type == reg_t || m_type == address_t || - m_type == memory_t || m_type == label_t); - return m_value.m_symbolic->name(); - } - - unsigned get_vect_nelem() const { - assert(is_vector()); - if (!m_value.m_vector_symbolic[0]) return 0; - if (!m_value.m_vector_symbolic[1]) return 1; - if (!m_value.m_vector_symbolic[2]) return 2; - if (!m_value.m_vector_symbolic[3]) return 3; - if (!m_value.m_vector_symbolic[4]) return 4; - if (!m_value.m_vector_symbolic[5]) return 5; - if (!m_value.m_vector_symbolic[6]) return 6; - if (!m_value.m_vector_symbolic[7]) return 7; - return 8; - } - - const symbol *vec_symbol(int idx) const { - assert(idx < 8); - const symbol *result = m_value.m_vector_symbolic[idx]; - assert(result != NULL); - return result; - } - - const std::string &vec_name1() const { - assert(m_type == vector_t); - return m_value.m_vector_symbolic[0]->name(); - } - - const std::string &vec_name2() const { - assert(m_type == vector_t); - return m_value.m_vector_symbolic[1]->name(); - } - - const std::string &vec_name3() const { - assert(m_type == vector_t); - return m_value.m_vector_symbolic[2]->name(); - } - - const std::string &vec_name4() const { - assert(m_type == vector_t); - return m_value.m_vector_symbolic[3]->name(); - } - - bool is_reg() const { - if (m_type == reg_t) { - return true; - } - if (m_type != symbolic_t) { - return false; - } - return m_value.m_symbolic->type()->get_key().is_reg(); - } - bool is_param_local() const { - if (m_type != symbolic_t) return false; - return m_value.m_symbolic->type()->get_key().is_param_local(); - } - - bool is_param_kernel() const { - if (m_type != symbolic_t) return false; - return m_value.m_symbolic->type()->get_key().is_param_kernel(); - } - - bool is_vector() const { - if (m_vector) return true; - return false; - } - int reg_num() const { return m_value.m_symbolic->reg_num(); } - int reg1_num() const { return m_value.m_vector_symbolic[0]->reg_num(); } - int reg2_num() const { return m_value.m_vector_symbolic[1]->reg_num(); } - int reg3_num() const { - return m_value.m_vector_symbolic[2] - ? m_value.m_vector_symbolic[2]->reg_num() - : 0; - } - int reg4_num() const { - return m_value.m_vector_symbolic[3] - ? m_value.m_vector_symbolic[3]->reg_num() - : 0; - } - int reg5_num() const { - return m_value.m_vector_symbolic[4] - ? m_value.m_vector_symbolic[4]->reg_num() - : 0; - } - int reg6_num() const { - return m_value.m_vector_symbolic[5] - ? m_value.m_vector_symbolic[5]->reg_num() - : 0; - } - int reg7_num() const { - return m_value.m_vector_symbolic[6] - ? m_value.m_vector_symbolic[6]->reg_num() - : 0; - } - int reg8_num() const { - return m_value.m_vector_symbolic[7] - ? m_value.m_vector_symbolic[7]->reg_num() - : 0; - } - int arch_reg_num() const { return m_value.m_symbolic->arch_reg_num(); } - int arch_reg_num(unsigned n) const { - return (m_value.m_vector_symbolic[n]) - ? m_value.m_vector_symbolic[n]->arch_reg_num() - : -1; - } - bool is_label() const { return m_type == label_t; } - bool is_builtin() const { return m_type == builtin_t; } - - // Memory operand used in ld / st instructions (ex. [__var1]) - bool is_memory_operand() const { return m_type == memory_t; } - - // Memory operand with immediate access (ex. s[0x0004] or g[$r1+=0x0004]) - // This is used by the PTXPlus extension. The operand is assigned an address - // space during parsing. - bool is_memory_operand2() const { return (m_addr_space != undefined_space); } - - bool is_immediate_address() const { return m_immediate_address; } - - bool is_literal() const { - return m_type == int_t || m_type == float_op_t || m_type == double_op_t || - m_type == unsigned_t; - } - bool is_shared() const { - if (!(m_type == symbolic_t || m_type == address_t || m_type == memory_t)) { - return false; - } - return m_value.m_symbolic->is_shared(); - } - bool is_sstarr() const { return m_value.m_symbolic->is_sstarr(); } - bool is_const() const { return m_value.m_symbolic->is_const(); } - bool is_global() const { return m_value.m_symbolic->is_global(); } - bool is_local() const { return m_value.m_symbolic->is_local(); } - bool is_tex() const { return m_value.m_symbolic->is_tex(); } - bool is_return_var() const { return m_is_return_var; } - - bool is_function_address() const { - if (m_type != symbolic_t) { +public: + operand_info(gpgpu_context* ctx) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = false; + m_immediate_address=false; + m_addr_offset = 0; + m_value.m_symbolic=NULL; + } + operand_info( const symbol *addr, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + if ( addr->is_label() ) { + m_type = label_t; + } else if ( addr->is_shared() ) { + m_type = symbolic_t; + } else if ( addr->is_const() ) { + m_type = symbolic_t; + } else if ( addr->is_global() ) { + m_type = symbolic_t; + } else if ( addr->is_local() ) { + m_type = symbolic_t; + } else if ( addr->is_param_local() ) { + m_type = symbolic_t; + } else if ( addr->is_param_kernel() ) { + m_type = symbolic_t; + } else if ( addr->is_tex() ) { + m_type = symbolic_t; + } else if ( addr->is_func_addr() ) { + m_type = symbolic_t; + } else if ( !addr->is_reg() ) { + m_type = symbolic_t; + } else { + m_type = reg_t; + } + + m_is_non_arch_reg = addr->is_non_arch_reg(); + m_value.m_symbolic = addr; + m_addr_offset = 0; + m_vector = false; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + operand_info( const symbol *addr1, const symbol *addr2, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_type = memory_t; + m_value.m_vector_symbolic = new const symbol*[8]; + m_value.m_vector_symbolic[0] = addr1; + m_value.m_vector_symbolic[1] = addr2; + m_value.m_vector_symbolic[2] = NULL; + m_value.m_vector_symbolic[3] = NULL; + m_value.m_vector_symbolic[4] = NULL; + m_value.m_vector_symbolic[5] = NULL; + m_value.m_vector_symbolic[6] = NULL; + m_value.m_vector_symbolic[7] = NULL; + m_addr_offset = 0; + m_vector = false; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + operand_info( int builtin_id, int dim_mod, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = builtin_t; + m_value.m_int = builtin_id; + m_addr_offset = dim_mod; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + operand_info( const symbol *addr, int offset, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = address_t; + m_value.m_symbolic = addr; + m_addr_offset = offset; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + operand_info( unsigned x, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = unsigned_t; + m_value.m_unsigned = x; + m_addr_offset = x; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=true; + } + operand_info( int x, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = int_t; + m_value.m_int = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + operand_info( float x, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = float_op_t; + m_value.m_float = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + operand_info( double x, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = double_op_t; + m_value.m_double = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = true; + m_type = vector_t; + m_value.m_vector_symbolic = new const symbol*[8]; + m_value.m_vector_symbolic[0] = s1; + m_value.m_vector_symbolic[1] = s2; + m_value.m_vector_symbolic[2] = s3; + m_value.m_vector_symbolic[3] = s4; + m_value.m_vector_symbolic[4] = NULL; + m_value.m_vector_symbolic[5] = NULL; + m_value.m_vector_symbolic[6] = NULL; + m_value.m_vector_symbolic[7] = NULL; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4 ,const symbol *s5,const symbol *s6,const symbol *s7, const symbol *s8, gpgpu_context* ctx) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = true; + m_type = vector_t; + m_value.m_vector_symbolic = new const symbol*[8]; + m_value.m_vector_symbolic[0] = s1; + m_value.m_vector_symbolic[1] = s2; + m_value.m_vector_symbolic[2] = s3; + m_value.m_vector_symbolic[3] = s4; + m_value.m_vector_symbolic[4] = s5; + m_value.m_vector_symbolic[5] = s6; + m_value.m_vector_symbolic[6] = s7; + m_value.m_vector_symbolic[7] = s8; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + + void init(gpgpu_context* ctx) + { + gpgpu_ctx = ctx; + m_uid=(unsigned)-1; + m_valid=false; + m_vector=false; + m_type=undef_t; + m_immediate_address=false; + m_addr_space=undefined_space; + m_operand_lohi=0; + m_double_operand_type=0; + m_operand_neg=false; + m_const_mem_offset=(unsigned)-1; + m_value.m_int=0; + m_value.m_unsigned=(unsigned)-1; + m_value.m_float=0; + m_value.m_double=0; + for(unsigned i=0; i<4; i++){ + m_value.m_vint[i]=0; + m_value.m_vunsigned[i]=0; + m_value.m_vfloat[i]=0; + m_value.m_vdouble[i]=0; + } + m_value.m_symbolic=NULL; + m_value.m_vector_symbolic=NULL; + m_addr_offset=0; + m_neg_pred=0; + m_is_return_var=0; + m_is_non_arch_reg=0; + + } + void make_memory_operand() { m_type = memory_t;} + void set_return() { m_is_return_var = true; } + void set_immediate_addr() {m_immediate_address=true;} + const std::string &name() const + { + assert( m_type == symbolic_t || m_type == reg_t || m_type == address_t || m_type == memory_t || m_type == label_t); + return m_value.m_symbolic->name(); + } + + unsigned get_vect_nelem() const + { + assert( is_vector() ); + if( !m_value.m_vector_symbolic[0] ) return 0; + if( !m_value.m_vector_symbolic[1] ) return 1; + if( !m_value.m_vector_symbolic[2] ) return 2; + if( !m_value.m_vector_symbolic[3] ) return 3; + if( !m_value.m_vector_symbolic[4] ) return 4; + if( !m_value.m_vector_symbolic[5] ) return 5; + if( !m_value.m_vector_symbolic[6] ) return 6; + if( !m_value.m_vector_symbolic[7] ) return 7; + return 8; + } + + const symbol* vec_symbol(int idx) const + { + assert(idx < 8); + const symbol *result = m_value.m_vector_symbolic[idx]; + assert( result != NULL ); + return result; + } + + const std::string &vec_name1() const + { + assert( m_type == vector_t); + return m_value.m_vector_symbolic[0]->name(); + } + + const std::string &vec_name2() const + { + assert( m_type == vector_t); + return m_value.m_vector_symbolic[1]->name(); + } + + const std::string &vec_name3() const + { + assert( m_type == vector_t); + return m_value.m_vector_symbolic[2]->name(); + } + + const std::string &vec_name4() const + { + assert( m_type == vector_t); + return m_value.m_vector_symbolic[3]->name(); + } + + bool is_reg() const + { + if ( m_type == reg_t ) { + return true; + } + if ( m_type != symbolic_t ) { + return false; + } + return m_value.m_symbolic->type()->get_key().is_reg(); + } + bool is_param_local() const + { + if ( m_type != symbolic_t ) + return false; + return m_value.m_symbolic->type()->get_key().is_param_local(); + } + + bool is_param_kernel() const + { + if ( m_type != symbolic_t ) + return false; + return m_value.m_symbolic->type()->get_key().is_param_kernel(); + } + + bool is_vector() const + { + if ( m_vector) return true; return false; - } - return m_value.m_symbolic->is_func_addr(); - } - - ptx_reg_t get_literal_value() const { - ptx_reg_t result; - switch (m_type) { - case int_t: - result.s64 = m_value.m_int; - break; - case float_op_t: - result.f32 = m_value.m_float; - break; - case double_op_t: - result.f64 = m_value.m_double; - break; - case unsigned_t: - result.u32 = m_value.m_unsigned; - break; + } + int reg_num() const { return m_value.m_symbolic->reg_num();} + int reg1_num() const { return m_value.m_vector_symbolic[0]->reg_num();} + int reg2_num() const { return m_value.m_vector_symbolic[1]->reg_num();} + int reg3_num() const { return m_value.m_vector_symbolic[2]?m_value.m_vector_symbolic[2]->reg_num():0; } + int reg4_num() const { return m_value.m_vector_symbolic[3]?m_value.m_vector_symbolic[3]->reg_num():0; } + int reg5_num() const { return m_value.m_vector_symbolic[4]?m_value.m_vector_symbolic[4]->reg_num():0; } + int reg6_num() const { return m_value.m_vector_symbolic[5]?m_value.m_vector_symbolic[5]->reg_num():0; } + int reg7_num() const { return m_value.m_vector_symbolic[6]?m_value.m_vector_symbolic[6]->reg_num():0; } + int reg8_num() const { return m_value.m_vector_symbolic[7]?m_value.m_vector_symbolic[7]->reg_num():0; } + int arch_reg_num() const { return m_value.m_symbolic->arch_reg_num(); } + int arch_reg_num(unsigned n) const { return (m_value.m_vector_symbolic[n])? m_value.m_vector_symbolic[n]->arch_reg_num() : -1; } + bool is_label() const { return m_type == label_t;} + bool is_builtin() const { return m_type == builtin_t;} + + // Memory operand used in ld / st instructions (ex. [__var1]) + bool is_memory_operand() const { return m_type == memory_t;} + + // Memory operand with immediate access (ex. s[0x0004] or g[$r1+=0x0004]) + // This is used by the PTXPlus extension. The operand is assigned an address space during parsing. + bool is_memory_operand2() const { + return (m_addr_space!=undefined_space); + } + + bool is_immediate_address() const { + return m_immediate_address; + } + + bool is_literal() const { return m_type == int_t || + m_type == float_op_t || + m_type == double_op_t || + m_type == unsigned_t;} + bool is_shared() const { + if ( !(m_type == symbolic_t || m_type == address_t || m_type == memory_t) ) { + return false; + } + return m_value.m_symbolic->is_shared(); + } + bool is_sstarr() const { return m_value.m_symbolic->is_sstarr();} + bool is_const() const { return m_value.m_symbolic->is_const();} + bool is_global() const { return m_value.m_symbolic->is_global();} + bool is_local() const { return m_value.m_symbolic->is_local();} + bool is_tex() const { return m_value.m_symbolic->is_tex();} + bool is_return_var() const { return m_is_return_var; } + + bool is_function_address() const + { + if( m_type != symbolic_t ) { + return false; + } + return m_value.m_symbolic->is_func_addr(); + } + + ptx_reg_t get_literal_value() const + { + ptx_reg_t result; + switch ( m_type ) { + case int_t: result.s64 = m_value.m_int; break; + case float_op_t: result.f32 = m_value.m_float; break; + case double_op_t: result.f64 = m_value.m_double; break; + case unsigned_t: result.u32 = m_value.m_unsigned; break; default: - assert(0); - break; - } - return result; - } - int get_int() const { return m_value.m_int; } - int get_addr_offset() const { return m_addr_offset; } - const symbol *get_symbol() const { return m_value.m_symbolic; } - void set_type(enum operand_type type) { m_type = type; } - enum operand_type get_type() const { return m_type; } - void set_neg_pred() { - assert(m_valid); - m_neg_pred = true; - } - bool is_neg_pred() const { return m_neg_pred; } - bool is_valid() const { return m_valid; } - - void set_addr_space(enum _memory_space_t set_value) { - m_addr_space = set_value; - } - enum _memory_space_t get_addr_space() const { return m_addr_space; } - void set_operand_lohi(int set_value) { m_operand_lohi = set_value; } - int get_operand_lohi() const { return m_operand_lohi; } - void set_double_operand_type(int set_value) { - m_double_operand_type = set_value; - } - int get_double_operand_type() const { return m_double_operand_type; } - void set_operand_neg() { m_operand_neg = true; } - bool get_operand_neg() const { return m_operand_neg; } - void set_const_mem_offset(addr_t set_value) { - m_const_mem_offset = set_value; - } - addr_t get_const_mem_offset() const { return m_const_mem_offset; } - bool is_non_arch_reg() const { return m_is_non_arch_reg; } - - private: - gpgpu_context *gpgpu_ctx; - unsigned m_uid; - bool m_valid; - bool m_vector; - enum operand_type m_type; - bool m_immediate_address; - enum _memory_space_t m_addr_space; - int m_operand_lohi; - int m_double_operand_type; - bool m_operand_neg; - addr_t m_const_mem_offset; - union { - int m_int; - unsigned int m_unsigned; - float m_float; - double m_double; - int m_vint[4]; - unsigned int m_vunsigned[4]; - float m_vfloat[4]; - double m_vdouble[4]; - const symbol *m_symbolic; - const symbol **m_vector_symbolic; - } m_value; - - int m_addr_offset; - - bool m_neg_pred; - bool m_is_return_var; - bool m_is_non_arch_reg; - - unsigned get_uid(); + assert(0); + break; + } + return result; + } + int get_int() const { return m_value.m_int;} + int get_addr_offset() const { return m_addr_offset;} + const symbol *get_symbol() const { return m_value.m_symbolic;} + void set_type( enum operand_type type ) + { + m_type = type; + } + enum operand_type get_type() const { + return m_type; + } + void set_neg_pred() + { + assert( m_valid ); + m_neg_pred = true; + } + bool is_neg_pred() const { return m_neg_pred; } + bool is_valid() const { return m_valid; } + + void set_addr_space(enum _memory_space_t set_value) { m_addr_space = set_value; } + enum _memory_space_t get_addr_space() const { return m_addr_space; } + void set_operand_lohi(int set_value) { m_operand_lohi = set_value; } + int get_operand_lohi() const { return m_operand_lohi; } + void set_double_operand_type(int set_value) { m_double_operand_type = set_value; } + int get_double_operand_type() const { return m_double_operand_type; } + void set_operand_neg() { m_operand_neg = true; } + bool get_operand_neg() const { return m_operand_neg; } + void set_const_mem_offset(addr_t set_value) { m_const_mem_offset = set_value; } + addr_t get_const_mem_offset() const { return m_const_mem_offset; } + bool is_non_arch_reg() const { return m_is_non_arch_reg; } + +private: + gpgpu_context* gpgpu_ctx; + unsigned m_uid; + bool m_valid; + bool m_vector; + enum operand_type m_type; + bool m_immediate_address; + enum _memory_space_t m_addr_space; + int m_operand_lohi; + int m_double_operand_type; + bool m_operand_neg; + addr_t m_const_mem_offset; + union { + int m_int; + unsigned int m_unsigned; + float m_float; + double m_double; + int m_vint[4]; + unsigned int m_vunsigned[4]; + float m_vfloat[4]; + double m_vdouble[4]; + const symbol* m_symbolic; + const symbol** m_vector_symbolic; + } m_value; + + int m_addr_offset; + + bool m_neg_pred; + bool m_is_return_var; + bool m_is_non_arch_reg; + + unsigned get_uid(); }; extern const char *g_opcode_string[]; struct basic_block_t { - basic_block_t(unsigned ID, ptx_instruction *begin, ptx_instruction *end, - bool entry, bool ex) { - bb_id = ID; - ptx_begin = begin; - ptx_end = end; - is_entry = entry; - is_exit = ex; - immediatepostdominator_id = -1; - immediatedominator_id = -1; - } - - ptx_instruction *ptx_begin; - ptx_instruction *ptx_end; - std::set - predecessor_ids; // indices of other basic blocks in m_basic_blocks array - std::set successor_ids; - std::set postdominator_ids; - std::set dominator_ids; - std::set Tmp_ids; - int immediatepostdominator_id; - int immediatedominator_id; - bool is_entry; - bool is_exit; - unsigned bb_id; - - // if this basic block dom B - bool dom(const basic_block_t *B) { - return (B->dominator_ids.find(this->bb_id) != B->dominator_ids.end()); - } - - // if this basic block pdom B - bool pdom(const basic_block_t *B) { - return (B->postdominator_ids.find(this->bb_id) != - B->postdominator_ids.end()); - } + basic_block_t( unsigned ID, ptx_instruction *begin, ptx_instruction *end, bool entry, bool ex) + { + bb_id = ID; + ptx_begin = begin; + ptx_end = end; + is_entry=entry; + is_exit=ex; + immediatepostdominator_id = -1; + immediatedominator_id = -1; + } + + ptx_instruction* ptx_begin; + ptx_instruction* ptx_end; + std::set predecessor_ids; //indices of other basic blocks in m_basic_blocks array + std::set successor_ids; + std::set postdominator_ids; + std::set dominator_ids; + std::set Tmp_ids; + int immediatepostdominator_id; + int immediatedominator_id; + bool is_entry; + bool is_exit; + unsigned bb_id; + + // if this basic block dom B + bool dom(const basic_block_t *B) { + return (B->dominator_ids.find(this->bb_id) != B->dominator_ids.end()); + } + + // if this basic block pdom B + bool pdom(const basic_block_t *B) { + return (B->postdominator_ids.find(this->bb_id) != B->postdominator_ids.end()); + } }; struct gpgpu_recon_t { - address_type source_pc; - address_type target_pc; - class ptx_instruction *source_inst; - class ptx_instruction *target_inst; + address_type source_pc; + address_type target_pc; + class ptx_instruction* source_inst; + class ptx_instruction* target_inst; }; class ptx_instruction : public warp_inst_t { - public: - ptx_instruction(int opcode, const symbol *pred, int neg_pred, int pred_mod, - symbol *label, const std::list &operands, - const operand_info &return_var, const std::list &options, - const std::list &wmma_options, - const std::list &scalar_type, memory_space_t space_spec, - const char *file, unsigned line, const char *source, - const core_config *config, gpgpu_context *ctx); - - void print_insn() const; - virtual void print_insn(FILE *fp) const; - std::string to_string() const; - unsigned inst_size() const { return m_inst_size; } - unsigned uid() const { return m_uid; } - int get_opcode() const { return m_opcode; } - const char *get_opcode_cstr() const { - if (m_opcode != -1) { - return g_opcode_string[m_opcode]; - } else { - return "label"; - } - } - const char *source_file() const { return m_source_file.c_str(); } - unsigned source_line() const { return m_source_line; } - unsigned get_num_operands() const { return m_operands.size(); } - bool has_pred() const { return m_pred != NULL; } - operand_info get_pred() const; - bool get_pred_neg() const { return m_neg_pred; } - int get_pred_mod() const { return m_pred_mod; } - const char *get_source() const { return m_source.c_str(); } - - typedef std::vector::const_iterator const_iterator; - - const_iterator op_iter_begin() const { return m_operands.begin(); } - - const_iterator op_iter_end() const { return m_operands.end(); } - - const operand_info &dst() const { - assert(!m_operands.empty()); - return m_operands[0]; - } - - const operand_info &func_addr() const { - assert(!m_operands.empty()); - if (!m_operands[0].is_return_var()) { +public: + ptx_instruction( int opcode, + const symbol *pred, + int neg_pred, + int pred_mod, + symbol *label, + const std::list &operands, + const operand_info &return_var, + const std::list &options, + const std::list &wmma_options, + const std::list &scalar_type, + memory_space_t space_spec, + const char *file, + unsigned line, + const char *source, + const core_config *config, + gpgpu_context* ctx); + + void print_insn() const; + virtual void print_insn( FILE *fp ) const; + std::string to_string() const; + unsigned inst_size() const { return m_inst_size; } + unsigned uid() const { return m_uid;} + int get_opcode() const { return m_opcode;} + const char *get_opcode_cstr() const + { + if ( m_opcode != -1 ) { + return g_opcode_string[m_opcode]; + } else { + return "label"; + } + } + const char *source_file() const { return m_source_file.c_str();} + unsigned source_line() const { return m_source_line;} + unsigned get_num_operands() const { return m_operands.size();} + bool has_pred() const { return m_pred != NULL;} + operand_info get_pred() const; + bool get_pred_neg() const { return m_neg_pred;} + int get_pred_mod() const { return m_pred_mod;} + const char *get_source() const { return m_source.c_str();} + + typedef std::vector::const_iterator const_iterator; + + const_iterator op_iter_begin() const + { + return m_operands.begin(); + } + + const_iterator op_iter_end() const + { + return m_operands.end(); + } + + const operand_info &dst() const + { + assert( !m_operands.empty() ); + return m_operands[0]; + } + + const operand_info &func_addr() const + { + assert( !m_operands.empty() ); + if( !m_operands[0].is_return_var() ) { + return m_operands[0]; + } else { + assert( m_operands.size() >= 2 ); + return m_operands[1]; + } + } + + operand_info &dst() + { + assert( !m_operands.empty() ); return m_operands[0]; - } else { - assert(m_operands.size() >= 2); + } + + const operand_info &src1() const + { + assert( m_operands.size() > 1 ); return m_operands[1]; - } - } - - operand_info &dst() { - assert(!m_operands.empty()); - return m_operands[0]; - } - - const operand_info &src1() const { - assert(m_operands.size() > 1); - return m_operands[1]; - } - - const operand_info &src2() const { - assert(m_operands.size() > 2); - return m_operands[2]; - } - - const operand_info &src3() const { - assert(m_operands.size() > 3); - return m_operands[3]; - } - const operand_info &src4() const { - assert(m_operands.size() > 4); - return m_operands[4]; - } - const operand_info &src5() const { - assert(m_operands.size() > 5); - return m_operands[5]; - } - const operand_info &src6() const { - assert(m_operands.size() > 6); - return m_operands[6]; - } - const operand_info &src7() const { - assert(m_operands.size() > 7); - return m_operands[7]; - } - const operand_info &src8() const { - assert(m_operands.size() > 8); - return m_operands[8]; - } - - const operand_info &operand_lookup(unsigned n) const { - assert(n < m_operands.size()); - return m_operands[n]; - } - bool has_return() const { return m_return_var.is_valid(); } - - memory_space_t get_space() const { return m_space_spec; } - unsigned get_vector() const { return m_vector_spec; } - unsigned get_atomic() const { return m_atomic_spec; } - - int get_wmma_type() const { return m_wmma_type; } - int get_wmma_layout(int index) const { - return m_wmma_layout[index]; // 0->Matrix D,1->Matrix C - } - int get_type() const { - assert(!m_scalar_type.empty()); - return m_scalar_type.front(); - } - - int get_type2() const { - assert(m_scalar_type.size() == 2); - return m_scalar_type.back(); - } - - void assign_bb( - basic_block_t *basic_block) // assign instruction to a basic block - { - m_basic_block = basic_block; - } - basic_block_t *get_bb() { return m_basic_block; } - void set_m_instr_mem_index(unsigned index) { m_instr_mem_index = index; } - void set_PC(addr_t PC) { m_PC = PC; } - addr_t get_PC() const { return m_PC; } - - unsigned get_m_instr_mem_index() { return m_instr_mem_index; } - unsigned get_cmpop() const { return m_compare_op; } - const symbol *get_label() const { return m_label; } - bool is_label() const { - if (m_label) { - assert(m_opcode == -1); - return true; - } - return false; - } - bool is_hi() const { return m_hi; } - bool is_lo() const { return m_lo; } - bool is_wide() const { return m_wide; } - bool is_uni() const { return m_uni; } - bool is_exit() const { return m_exit; } - bool is_abs() const { return m_abs; } - bool is_neg() const { return m_neg; } - bool is_to() const { return m_to_option; } - unsigned cache_option() const { return m_cache_option; } - unsigned rounding_mode() const { return m_rounding_mode; } - unsigned saturation_mode() const { return m_saturation_mode; } - unsigned dimension() const { return m_geom_spec; } - unsigned barrier_op() const { return m_barrier_op; } - unsigned shfl_op() const { return m_shfl_op; } - unsigned prmt_op() const { return m_prmt_op; } - enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot }; - enum vote_mode_t vote_mode() const { return m_vote_mode; } - - int membar_level() const { return m_membar_level; } - - bool has_memory_read() const { - if (m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP || - m_opcode == MMA_LD_OP) - return true; - // Check PTXPlus operand type below - // Source operands are memory operands - ptx_instruction::const_iterator op = op_iter_begin(); - for (int n = 0; op != op_iter_end(); op++, n++) { // process operands - if (n > 0 && op->is_memory_operand2()) // source operands only - return true; - } - return false; - } - bool has_memory_write() const { - if (m_opcode == ST_OP || m_opcode == MMA_ST_OP) return true; - // Check PTXPlus operand type below - // Destination operand is a memory operand - ptx_instruction::const_iterator op = op_iter_begin(); - for (int n = 0; (op != op_iter_end() && n < 1); - op++, n++) { // process operands - if (n == 0 && op->is_memory_operand2()) // source operands only - return true; - } - return false; - } - - private: - void set_opcode_and_latency(); - void set_bar_type(); - void set_fp_or_int_archop(); - void set_mul_div_or_other_archop(); - - basic_block_t *m_basic_block; - unsigned m_uid; - addr_t m_PC; - std::string m_source_file; - unsigned m_source_line; - std::string m_source; - - const symbol *m_pred; - bool m_neg_pred; - int m_pred_mod; - int m_opcode; - const symbol *m_label; - std::vector m_operands; - operand_info m_return_var; - - std::list m_options; - std::list m_wmma_options; - bool m_wide; - bool m_hi; - bool m_lo; - bool m_exit; - bool m_abs; - bool m_neg; - bool m_uni; // if branch instruction, this evaluates to true for uniform - // branches (ie jumps) - bool m_to_option; - unsigned m_cache_option; - int m_wmma_type; - int m_wmma_layout[2]; - int m_wmma_configuration; - unsigned m_rounding_mode; - unsigned m_compare_op; - unsigned m_saturation_mode; - unsigned m_barrier_op; - unsigned m_shfl_op; - unsigned m_prmt_op; - - std::list m_scalar_type; - memory_space_t m_space_spec; - int m_geom_spec; - int m_vector_spec; - int m_atomic_spec; - enum vote_mode_t m_vote_mode; - int m_membar_level; - int m_instr_mem_index; // index into m_instr_mem array - unsigned m_inst_size; // bytes - - virtual void pre_decode(); - friend class function_info; - // backward pointer - class gpgpu_context *gpgpu_ctx; + } + + const operand_info &src2() const + { + assert( m_operands.size() > 2 ); + return m_operands[2]; + } + + const operand_info &src3() const + { + assert( m_operands.size() > 3 ); + return m_operands[3]; + } + const operand_info &src4() const + { + assert( m_operands.size() > 4 ); + return m_operands[4]; + } + const operand_info &src5() const + { + assert( m_operands.size() > 5 ); + return m_operands[5]; + } + const operand_info &src6() const + { + assert( m_operands.size() > 6 ); + return m_operands[6]; + } + const operand_info &src7() const + { + assert( m_operands.size() > 7 ); + return m_operands[7]; + } + const operand_info &src8() const + { + assert( m_operands.size() > 8 ); + return m_operands[8]; + } + + const operand_info &operand_lookup( unsigned n ) const + { + assert( n < m_operands.size() ); + return m_operands[n]; + } + bool has_return() const + { + return m_return_var.is_valid(); + } + + memory_space_t get_space() const { return m_space_spec;} + unsigned get_vector() const { return m_vector_spec;} + unsigned get_atomic() const { return m_atomic_spec;} + + int get_wmma_type() const { + return m_wmma_type; + } + int get_wmma_layout(int index) const { + return m_wmma_layout[index];//0->Matrix D,1->Matrix C + } + int get_type() const + { + assert( !m_scalar_type.empty() ); + return m_scalar_type.front(); + } + + int get_type2() const + { + assert( m_scalar_type.size()==2 ); + return m_scalar_type.back(); + } + + void assign_bb(basic_block_t* basic_block) //assign instruction to a basic block + { + m_basic_block = basic_block; + } + basic_block_t* get_bb() { return m_basic_block;} + void set_m_instr_mem_index(unsigned index) { + m_instr_mem_index = index; + } + void set_PC( addr_t PC ) + { + m_PC = PC; + } + addr_t get_PC() const + { + return m_PC; + } + + unsigned get_m_instr_mem_index() { return m_instr_mem_index;} + unsigned get_cmpop() const { return m_compare_op;} + const symbol *get_label() const { return m_label;} + bool is_label() const { if(m_label){ assert(m_opcode==-1);return true;} return false;} + bool is_hi() const { return m_hi;} + bool is_lo() const { return m_lo;} + bool is_wide() const { return m_wide;} + bool is_uni() const { return m_uni;} + bool is_exit() const { return m_exit;} + bool is_abs() const { return m_abs;} + bool is_neg() const { return m_neg;} + bool is_to() const { return m_to_option; } + unsigned cache_option() const { return m_cache_option; } + unsigned rounding_mode() const { return m_rounding_mode;} + unsigned saturation_mode() const { return m_saturation_mode;} + unsigned dimension() const { return m_geom_spec;} + unsigned barrier_op() const {return m_barrier_op;} + unsigned shfl_op() const {return m_shfl_op;} + unsigned prmt_op() const {return m_prmt_op;} + enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot }; + enum vote_mode_t vote_mode() const { return m_vote_mode; } + + int membar_level() const { return m_membar_level; } + + bool has_memory_read() const { + if( m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP|| m_opcode==MMA_LD_OP ) + return true; + // Check PTXPlus operand type below + // Source operands are memory operands + ptx_instruction::const_iterator op=op_iter_begin(); + for ( int n=0; op != op_iter_end(); op++, n++ ) { //process operands + if( n > 0 && op->is_memory_operand2()) // source operands only + return true; + } + return false; + } + bool has_memory_write() const { + if( m_opcode == ST_OP || m_opcode==MMA_ST_OP ) return true; + // Check PTXPlus operand type below + // Destination operand is a memory operand + ptx_instruction::const_iterator op=op_iter_begin(); + for ( int n=0; (op!=op_iter_end() && n<1); op++, n++ ) { //process operands + if( n==0 && op->is_memory_operand2()) // source operands only + return true; + } + return false; + } + + +private: + void set_opcode_and_latency(); + void set_bar_type(); + void set_fp_or_int_archop(); + void set_mul_div_or_other_archop(); + + basic_block_t *m_basic_block; + unsigned m_uid; + addr_t m_PC; + std::string m_source_file; + unsigned m_source_line; + std::string m_source; + + const symbol *m_pred; + bool m_neg_pred; + int m_pred_mod; + int m_opcode; + const symbol *m_label; + std::vector m_operands; + operand_info m_return_var; + + std::list m_options; + std::list m_wmma_options; + bool m_wide; + bool m_hi; + bool m_lo; + bool m_exit; + bool m_abs; + bool m_neg; + bool m_uni; //if branch instruction, this evaluates to true for uniform branches (ie jumps) + bool m_to_option; + unsigned m_cache_option; + int m_wmma_type; + int m_wmma_layout[2]; + int m_wmma_configuration; + unsigned m_rounding_mode; + unsigned m_compare_op; + unsigned m_saturation_mode; + unsigned m_barrier_op; + unsigned m_shfl_op; + unsigned m_prmt_op; + + std::list m_scalar_type; + memory_space_t m_space_spec; + int m_geom_spec; + int m_vector_spec; + int m_atomic_spec; + enum vote_mode_t m_vote_mode; + int m_membar_level; + int m_instr_mem_index; //index into m_instr_mem array + unsigned m_inst_size; // bytes + + virtual void pre_decode(); + friend class function_info; + // backward pointer + class gpgpu_context* gpgpu_ctx; }; class param_info { - public: - param_info() { - m_valid = false; - m_value_set = false; - m_size = 0; - m_is_ptr = false; - } - param_info(std::string name, int type, size_t size, bool is_ptr, - memory_space_t ptr_space) { - m_valid = true; - m_value_set = false; - m_name = name; - m_type = type; - m_size = size; - m_is_ptr = is_ptr; - m_ptr_space = ptr_space; - } - void add_data(param_t v) { - assert((!m_value_set) || (m_value.size == v.size)); // if this fails - // concurrent kernel - // launches might - // execute incorrectly - m_value_set = true; - m_value = v; - } - void add_offset(unsigned offset) { m_offset = offset; } - unsigned get_offset() { - assert(m_valid); - return m_offset; - } - std::string get_name() const { - assert(m_valid); - return m_name; - } - int get_type() const { - assert(m_valid); - return m_type; - } - param_t get_value() const { - assert(m_value_set); - return m_value; - } - size_t get_size() const { - assert(m_valid); - return m_size; - } - bool is_ptr_shared() const { - assert(m_valid); - return (m_is_ptr and m_ptr_space == shared_space); - } - - private: - bool m_valid; - std::string m_name; - int m_type; - size_t m_size; - bool m_value_set; - param_t m_value; - unsigned m_offset; - bool m_is_ptr; - memory_space_t m_ptr_space; +public: + param_info() { m_valid = false; m_value_set=false; m_size = 0; m_is_ptr = false; } + param_info( std::string name, int type, size_t size, bool is_ptr, memory_space_t ptr_space ) + { + m_valid = true; + m_value_set = false; + m_name = name; + m_type = type; + m_size = size; + m_is_ptr = is_ptr; + m_ptr_space = ptr_space; + } + void add_data( param_t v ) { + assert( (!m_value_set) || (m_value.size == v.size) ); // if this fails concurrent kernel launches might execute incorrectly + m_value_set = true; + m_value = v; + } + void add_offset( unsigned offset ) { m_offset = offset; } + unsigned get_offset() { assert(m_valid); return m_offset; } + std::string get_name() const { assert(m_valid); return m_name; } + int get_type() const { assert(m_valid); return m_type; } + param_t get_value() const { assert(m_value_set); return m_value; } + size_t get_size() const { assert(m_valid); return m_size; } + bool is_ptr_shared() const { assert(m_valid); return (m_is_ptr and m_ptr_space == shared_space); } +private: + bool m_valid; + std::string m_name; + int m_type; + size_t m_size; + bool m_value_set; + param_t m_value; + unsigned m_offset; + bool m_is_ptr; + memory_space_t m_ptr_space; }; class function_info { - public: - function_info(int entry_point, gpgpu_context *ctx); - const ptx_version &get_ptx_version() const { - return m_symtab->get_ptx_version(); - } - unsigned get_sm_target() const { return m_symtab->get_sm_target(); } - bool is_extern() const { return m_extern; } - void set_name(const char *name) { m_name = name; } - void set_symtab(symbol_table *symtab) { m_symtab = symtab; } - std::string get_name() const { return m_name; } - unsigned print_insn(unsigned pc, FILE *fp) const; - std::string get_insn_str(unsigned pc) const; - void add_inst(const std::list &instructions) { - m_instructions = instructions; - } - std::list::iterator find_next_real_instruction( - std::list::iterator i); - void create_basic_blocks(); - - void print_basic_blocks(); - - void print_basic_block_links(); - void print_basic_block_dot(); - - operand_info *find_break_target( - ptx_instruction *p_break_insn); // find the target of a break instruction - void connect_basic_blocks(); // iterate across m_basic_blocks of function, - // connecting basic blocks together - bool - connect_break_targets(); // connecting break instructions with proper targets - - // iterate across m_basic_blocks of function, - // finding dominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - void find_dominators(); - void print_dominators(); - void find_idominators(); - void print_idominators(); - - // iterate across m_basic_blocks of function, - // finding postdominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - void find_postdominators(); - void print_postdominators(); - - // iterate across m_basic_blocks of function, - // finding immediate postdominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - void find_ipostdominators(); - void print_ipostdominators(); - void do_pdom(); // function to call pdom analysis - - unsigned get_num_reconvergence_pairs(); - - void get_reconvergence_pairs(gpgpu_recon_t *recon_points); - - unsigned get_function_size() { return m_instructions.size(); } - - void ptx_assemble(); - - unsigned ptx_get_inst_op(ptx_thread_info *thread); - void add_param(const char *name, struct param_t value) { - m_kernel_params[name] = value; - } - void add_param_name_type_size(unsigned index, std::string name, int type, - size_t size, bool ptr, memory_space_t space); - void add_param_data(unsigned argn, struct gpgpu_ptx_sim_arg *args); - void add_return_var(const symbol *rv) { m_return_var_sym = rv; } - void add_arg(const symbol *arg) { - assert(arg != NULL); - m_args.push_back(arg); - } - void remove_args() { m_args.clear(); } - unsigned num_args() const { return m_args.size(); } - unsigned get_args_aligned_size(); - - const symbol *get_arg(unsigned n) const { - assert(n < m_args.size()); - return m_args[n]; - } - bool has_return() const { return m_return_var_sym != NULL; } - const symbol *get_return_var() const { return m_return_var_sym; } - const ptx_instruction *get_instruction(unsigned PC) const { - unsigned index = PC - m_start_PC; - if (index < m_instr_mem_size) return m_instr_mem[index]; - return NULL; - } - addr_t get_start_PC() const { return m_start_PC; } - - void finalize(memory_space *param_mem); - void param_to_shared(memory_space *shared_mem, symbol_table *symtab); - void list_param(FILE *fout) const; - void ptx_jit_config(std::map mallocPtr_Size, - memory_space *param_mem, gpgpu_t *gpu, dim3 gridDim, - dim3 blockDim); - - const struct gpgpu_ptx_sim_info *get_kernel_info() const { - assert(m_kernel_info.maxthreads == maxnt_id); - return &m_kernel_info; - } - - const void set_kernel_info(const struct gpgpu_ptx_sim_info &info) { - m_kernel_info = info; - m_kernel_info.ptx_version = 10 * get_ptx_version().ver(); - m_kernel_info.sm_target = get_ptx_version().target(); - // THIS DEPENDS ON ptxas being called after the PTX is parsed. - m_kernel_info.maxthreads = maxnt_id; - } - symbol_table *get_symtab() { return m_symtab; } - - unsigned local_mem_framesize() const { return m_local_mem_framesize; } - void set_framesize(unsigned sz) { m_local_mem_framesize = sz; } - bool is_entry_point() const { return m_entry_point; } - bool is_pdom_set() const { return pdom_done; } // return pdom flag - void set_pdom() { pdom_done = true; } // set pdom flag - - void add_config_param(size_t size, unsigned alignment) { - unsigned offset = 0; - if (m_param_configs.size() > 0) { - unsigned offset_nom = - m_param_configs.back().first + m_param_configs.back().second; - // ensure offset matches alignment requirements - offset = offset_nom % alignment ? (offset_nom / alignment + 1) * alignment - : offset_nom; - } - m_param_configs.push_back(std::pair(size, offset)); - } - - std::pair get_param_config(unsigned param_num) const { - return m_param_configs[param_num]; - } - - void set_maxnt_id(unsigned maxthreads) { maxnt_id = maxthreads; } - unsigned get_maxnt_id() { return maxnt_id; } - // backward pointer - class gpgpu_context *gpgpu_ctx; - - private: - unsigned maxnt_id; - unsigned m_uid; - unsigned m_local_mem_framesize; - bool m_entry_point; - bool m_extern; - bool m_assembled; - bool pdom_done; // flag to check whether pdom is completed or not - std::string m_name; - ptx_instruction **m_instr_mem; - unsigned m_start_PC; - unsigned m_instr_mem_size; - std::map m_kernel_params; - std::map m_ptx_kernel_param_info; - std::vector > m_param_configs; - const symbol *m_return_var_sym; - std::vector m_args; - std::list m_instructions; - std::vector m_basic_blocks; - std::list > m_back_edges; - std::map labels; - unsigned num_reconvergence_pairs; - - // Registers/shmem/etc. used (from ptxas -v), loaded from ___.ptxinfo along - // with ___.ptx - struct gpgpu_ptx_sim_info m_kernel_info; - - symbol_table *m_symtab; - - // parameter size for device kernels - int m_args_aligned_size; - - addr_t m_n; // offset in m_instr_mem (used in do_pdom) +public: + function_info(int entry_point, gpgpu_context* ctx ); + const ptx_version &get_ptx_version() const { return m_symtab->get_ptx_version(); } + unsigned get_sm_target() const { return m_symtab->get_sm_target(); } + bool is_extern() const { return m_extern; } + void set_name(const char *name) + { + m_name = name; + } + void set_symtab(symbol_table *symtab ) + { + m_symtab = symtab; + } + std::string get_name() const + { + return m_name; + } + unsigned print_insn( unsigned pc, FILE * fp ) const; + std::string get_insn_str( unsigned pc ) const; + void add_inst( const std::list &instructions ) + { + m_instructions = instructions; + } + std::list::iterator find_next_real_instruction( std::list::iterator i ); + void create_basic_blocks( ); + + void print_basic_blocks(); + + void print_basic_block_links(); + void print_basic_block_dot(); + + operand_info* find_break_target( ptx_instruction * p_break_insn ); //find the target of a break instruction + void connect_basic_blocks( ); //iterate across m_basic_blocks of function, connecting basic blocks together + bool connect_break_targets(); //connecting break instructions with proper targets + + //iterate across m_basic_blocks of function, + //finding dominator blocks, using algorithm of + //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 + void find_dominators( ); + void print_dominators(); + void find_idominators(); + void print_idominators(); + + //iterate across m_basic_blocks of function, + //finding postdominator blocks, using algorithm of + //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 + void find_postdominators( ); + void print_postdominators(); + + //iterate across m_basic_blocks of function, + //finding immediate postdominator blocks, using algorithm of + //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + void find_ipostdominators( ); + void print_ipostdominators(); + void do_pdom(); //function to call pdom analysis + + unsigned get_num_reconvergence_pairs(); + + void get_reconvergence_pairs(gpgpu_recon_t* recon_points); + + unsigned get_function_size() { return m_instructions.size();} + + void ptx_assemble(); + + unsigned ptx_get_inst_op( ptx_thread_info *thread ); + void add_param( const char *name, struct param_t value ) + { + m_kernel_params[ name ] = value; + } + void add_param_name_type_size( unsigned index, std::string name, int type, size_t size, bool ptr, memory_space_t space ); + void add_param_data( unsigned argn, struct gpgpu_ptx_sim_arg *args ); + void add_return_var( const symbol *rv ) + { + m_return_var_sym = rv; + } + void add_arg( const symbol *arg ) + { + assert( arg != NULL ); + m_args.push_back(arg); + } + void remove_args() + { + m_args.clear(); + } + unsigned num_args() const + { + return m_args.size(); + } + unsigned get_args_aligned_size(); + + const symbol* get_arg( unsigned n ) const + { + assert( n < m_args.size() ); + return m_args[n]; + } + bool has_return() const + { + return m_return_var_sym != NULL; + } + const symbol *get_return_var() const + { + return m_return_var_sym; + } + const ptx_instruction *get_instruction( unsigned PC ) const + { + unsigned index = PC - m_start_PC; + if( index < m_instr_mem_size ) + return m_instr_mem[index]; + return NULL; + } + addr_t get_start_PC() const + { + return m_start_PC; + } + + void finalize( memory_space *param_mem ); + void param_to_shared( memory_space *shared_mem, symbol_table *symtab ); + void list_param( FILE *fout ) const; + void ptx_jit_config(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) ; + + const struct gpgpu_ptx_sim_info* get_kernel_info () const + { + assert (m_kernel_info.maxthreads == maxnt_id); + return &m_kernel_info; + } + + const void set_kernel_info (const struct gpgpu_ptx_sim_info &info) { + m_kernel_info = info; + m_kernel_info.ptx_version = 10*get_ptx_version().ver(); + m_kernel_info.sm_target = get_ptx_version().target(); + // THIS DEPENDS ON ptxas being called after the PTX is parsed. + m_kernel_info.maxthreads = maxnt_id; + } + symbol_table *get_symtab() + { + return m_symtab; + } + + unsigned local_mem_framesize() const + { + return m_local_mem_framesize; + } + void set_framesize( unsigned sz ) + { + m_local_mem_framesize = sz; + } + bool is_entry_point() const { return m_entry_point; } + bool is_pdom_set() const { return pdom_done; } //return pdom flag + void set_pdom() { pdom_done = true; } //set pdom flag + + void add_config_param( size_t size, unsigned alignment ){ + unsigned offset = 0; + if (m_param_configs.size()>0){ + unsigned offset_nom = m_param_configs.back().first + m_param_configs.back().second; + //ensure offset matches alignment requirements + offset = offset_nom%alignment ? (offset_nom/alignment + 1) * alignment : offset_nom; + } + m_param_configs.push_back(std::pair(size, offset)); + } + + std::pair get_param_config(unsigned param_num) const { return m_param_configs[param_num]; } + + void set_maxnt_id(unsigned maxthreads) { maxnt_id = maxthreads;} + unsigned get_maxnt_id() { return maxnt_id;} + // backward pointer + class gpgpu_context* gpgpu_ctx; + +private: + unsigned maxnt_id; + unsigned m_uid; + unsigned m_local_mem_framesize; + bool m_entry_point; + bool m_extern; + bool m_assembled; + bool pdom_done; //flag to check whether pdom is completed or not + std::string m_name; + ptx_instruction **m_instr_mem; + unsigned m_start_PC; + unsigned m_instr_mem_size; + std::map m_kernel_params; + std::map m_ptx_kernel_param_info; + std::vector< std::pair > m_param_configs; + const symbol *m_return_var_sym; + std::vector m_args; + std::list m_instructions; + std::vector m_basic_blocks; + std::list > m_back_edges; + std::map labels; + unsigned num_reconvergence_pairs; + + //Registers/shmem/etc. used (from ptxas -v), loaded from ___.ptxinfo along with ___.ptx + struct gpgpu_ptx_sim_info m_kernel_info; + + symbol_table *m_symtab; + + //parameter size for device kernels + int m_args_aligned_size; + + addr_t m_n; // offset in m_instr_mem (used in do_pdom) }; class arg_buffer_t { - public: - arg_buffer_t(gpgpu_context *ctx) : m_src_op(ctx) { - m_is_reg = false; - m_is_param = false; - m_param_value = NULL; - m_reg_value = ptx_reg_t(); - } - arg_buffer_t(const arg_buffer_t &another, gpgpu_context *ctx) - : m_src_op(ctx) { - make_copy(another); - } - void make_copy(const arg_buffer_t &another) { - m_dst = another.m_dst; - m_src_op = another.m_src_op; - m_is_reg = another.m_is_reg; - m_is_param = another.m_is_param; - m_reg_value = another.m_reg_value; - m_param_bytes = another.m_param_bytes; - if (m_is_param) { - m_param_value = malloc(m_param_bytes); - memcpy(m_param_value, another.m_param_value, m_param_bytes); - } - } - void operator=(const arg_buffer_t &another) { make_copy(another); } - ~arg_buffer_t() { - if (m_is_param) free(m_param_value); - } - arg_buffer_t(const symbol *dst_sym, const operand_info &src_op, - ptx_reg_t source_value) - : m_src_op(src_op) { - m_dst = dst_sym; - m_reg_value = ptx_reg_t(); - if (dst_sym->is_reg()) { - m_is_reg = true; - m_is_param = false; - assert(src_op.is_reg()); - m_reg_value = source_value; - } else { - m_is_param = true; - m_is_reg = false; - m_param_value = calloc(sizeof(ptx_reg_t), 1); - // new (m_param_value) ptx_reg_t(source_value); - memcpy(m_param_value, &source_value, sizeof(ptx_reg_t)); - m_param_bytes = sizeof(ptx_reg_t); - } - } - arg_buffer_t(const symbol *dst_sym, const operand_info &src_op, - void *source_param_value_array, unsigned array_size) - : m_src_op(src_op) { - m_dst = dst_sym; - if (dst_sym->is_reg()) { - m_is_reg = true; - m_is_param = false; - assert(src_op.is_param_local()); - assert(dst_sym->get_size_in_bytes() == array_size); - switch (array_size) { - case 1: - m_reg_value.u8 = *(unsigned char *)source_param_value_array; - break; - case 2: - m_reg_value.u16 = *(unsigned short *)source_param_value_array; - break; - case 4: - m_reg_value.u32 = *(unsigned int *)source_param_value_array; - break; - case 8: - m_reg_value.u64 = *(unsigned long long *)source_param_value_array; - break; - default: - printf( - "GPGPU-Sim PTX: ERROR ** source param size does not match known " - "register sizes\n"); - break; +public: + arg_buffer_t(gpgpu_context* ctx) : m_src_op(ctx) + { + m_is_reg=false; + m_is_param=false; + m_param_value=NULL; + m_reg_value=ptx_reg_t(); + } + arg_buffer_t( const arg_buffer_t &another, gpgpu_context* ctx ) : m_src_op(ctx) + { + make_copy(another); + } + void make_copy( const arg_buffer_t &another ) + { + m_dst = another.m_dst; + m_src_op = another.m_src_op; + m_is_reg = another.m_is_reg; + m_is_param = another.m_is_param; + m_reg_value = another.m_reg_value; + m_param_bytes = another.m_param_bytes; + if( m_is_param ) { + m_param_value = malloc(m_param_bytes); + memcpy(m_param_value,another.m_param_value,m_param_bytes); + } + } + void operator=( const arg_buffer_t &another ) + { + make_copy(another); + } + ~arg_buffer_t() + { + if( m_is_param ) + free(m_param_value); + } + arg_buffer_t( const symbol *dst_sym, const operand_info &src_op, ptx_reg_t source_value ) : m_src_op(src_op) + { + m_dst = dst_sym; + m_reg_value=ptx_reg_t(); + if( dst_sym->is_reg() ) { + m_is_reg = true; + m_is_param = false; + assert( src_op.is_reg() ); + m_reg_value = source_value; + } else { + m_is_param = true; + m_is_reg = false; + m_param_value = calloc(sizeof(ptx_reg_t),1); + //new (m_param_value) ptx_reg_t(source_value); + memcpy(m_param_value,&source_value,sizeof(ptx_reg_t)); + m_param_bytes = sizeof(ptx_reg_t); } - } else { - // param - m_is_param = true; - m_is_reg = false; - m_param_value = calloc(array_size, 1); - m_param_bytes = array_size; - memcpy(m_param_value, source_param_value_array, array_size); - } - } - - bool is_reg() const { return m_is_reg; } - ptx_reg_t get_reg() const { - assert(m_is_reg); - return m_reg_value; - } - - const void *get_param_buffer() const { - assert(m_is_param); - return m_param_value; - } - size_t get_param_buffer_size() const { - assert(m_is_param); - return m_param_bytes; - } - - const symbol *get_dst() const { return m_dst; } - - private: - // destination of copy - const symbol *m_dst; - - // source operand - operand_info m_src_op; - - // source information - bool m_is_reg; - bool m_is_param; - - // source is register - ptx_reg_t m_reg_value; - - // source is param - void *m_param_value; - unsigned m_param_bytes; + } + arg_buffer_t( const symbol *dst_sym, const operand_info &src_op, void *source_param_value_array, unsigned array_size ) : m_src_op(src_op) + { + m_dst = dst_sym; + if( dst_sym->is_reg() ) { + m_is_reg = true; + m_is_param = false; + assert( src_op.is_param_local() ); + assert( dst_sym->get_size_in_bytes() == array_size ); + switch( array_size ) { + case 1: m_reg_value.u8 = *(unsigned char*)source_param_value_array; break; + case 2: m_reg_value.u16 = *(unsigned short*)source_param_value_array; break; + case 4: m_reg_value.u32 = *(unsigned int*)source_param_value_array; break; + case 8: m_reg_value.u64 = *(unsigned long long*)source_param_value_array; break; + default: + printf("GPGPU-Sim PTX: ERROR ** source param size does not match known register sizes\n"); + break; + } + } else { + // param + m_is_param = true; + m_is_reg = false; + m_param_value = calloc(array_size,1); + m_param_bytes = array_size; + memcpy(m_param_value,source_param_value_array,array_size); + } + } + + bool is_reg() const { return m_is_reg; } + ptx_reg_t get_reg() const + { + assert(m_is_reg); + return m_reg_value; + } + + const void *get_param_buffer() const + { + assert(m_is_param); + return m_param_value; + } + size_t get_param_buffer_size() const + { + assert(m_is_param); + return m_param_bytes; + } + + const symbol *get_dst() const { return m_dst; } + +private: + // destination of copy + const symbol *m_dst; + + // source operand + operand_info m_src_op; + + // source information + bool m_is_reg; + bool m_is_param; + + // source is register + ptx_reg_t m_reg_value; + + // source is param + void *m_param_value; + unsigned m_param_bytes; }; -typedef std::list arg_buffer_list_t; -arg_buffer_t copy_arg_to_buffer(ptx_thread_info *thread, - operand_info actual_param_op, - const symbol *formal_param); -void copy_args_into_buffer_list(const ptx_instruction *pI, - ptx_thread_info *thread, - const function_info *target_func, - arg_buffer_list_t &arg_values); -void copy_buffer_list_into_frame(ptx_thread_info *thread, - arg_buffer_list_t &arg_values); -void copy_buffer_to_frame(ptx_thread_info *thread, const arg_buffer_t &a); +typedef std::list< arg_buffer_t > arg_buffer_list_t; +arg_buffer_t copy_arg_to_buffer(ptx_thread_info * thread, operand_info actual_param_op, const symbol * formal_param); +void copy_args_into_buffer_list( const ptx_instruction * pI, + ptx_thread_info * thread, + const function_info * target_func, + arg_buffer_list_t &arg_values ); +void copy_buffer_list_into_frame(ptx_thread_info * thread, arg_buffer_list_t &arg_values); +void copy_buffer_to_frame(ptx_thread_info * thread, const arg_buffer_t &a); + struct textureInfo { - unsigned int texel_size; // size in bytes, e.g. (channelDesc.x+y+z+w)/8 - unsigned int Tx, - Ty; // tiling factor dimensions of layout of texels per 64B cache block - unsigned int Tx_numbits, Ty_numbits; // log2(T) - unsigned int texel_size_numbits; // log2(texel_size) + unsigned int texel_size; //size in bytes, e.g. (channelDesc.x+y+z+w)/8 + unsigned int Tx,Ty; //tiling factor dimensions of layout of texels per 64B cache block + unsigned int Tx_numbits,Ty_numbits; //log2(T) + unsigned int texel_size_numbits; //log2(texel_size) }; -extern std::map g_sym_name_to_symbol_table; +extern std::map g_sym_name_to_symbol_table; + -void gpgpu_ptx_assemble(std::string kname, void *kinfo); +void gpgpu_ptx_assemble( std::string kname, void *kinfo ); #include "../option_parser.h" -unsigned ptx_kernel_shmem_size(void *kernel_impl); -unsigned ptx_kernel_nregs(void *kernel_impl); +unsigned ptx_kernel_shmem_size( void *kernel_impl ); +unsigned ptx_kernel_nregs( void *kernel_impl ); #endif diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc index af66314..33bcf45 100644 --- a/src/cuda-sim/ptx_loader.cc +++ b/src/cuda-sim/ptx_loader.cc @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -28,556 +26,503 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "ptx_loader.h" -#include +#include "ptx_ir.h" +#include "cuda-sim.h" +#include "ptx_parser.h" #include +#include #include #include #include "../../libcuda/gpgpu_context.h" -#include "cuda-sim.h" -#include "ptx_ir.h" -#include "ptx_parser.h" /// extern prototypes -extern int ptx_error(yyscan_t yyscanner, const char *s); -extern int ptx_lex_init(yyscan_t *scanner); -extern void ptx_set_in(FILE *_in_str, yyscan_t yyscanner); -extern int ptx_parse(yyscan_t scanner, ptx_recognizer *recognizer); +extern int ptx_error( yyscan_t yyscanner, const char *s ); +extern int ptx_lex_init(yyscan_t* scanner); +extern void ptx_set_in(FILE * _in_str ,yyscan_t yyscanner ); +extern int ptx_parse(yyscan_t scanner, ptx_recognizer* recognizer); extern int ptx_lex_destroy(yyscan_t scanner); -extern int ptx__scan_string(const char *, yyscan_t scanner); +extern int ptx__scan_string(const char*, yyscan_t scanner); -extern std::map get_duplicate(); +extern std::map get_duplicate(); -typedef void *yyscan_t; -extern int ptxinfo_lex_init(yyscan_t *scanner); -extern void ptxinfo_set_in(FILE *_in_str, yyscan_t yyscanner); -extern int ptxinfo_parse(yyscan_t scanner, ptxinfo_data *ptxinfo); +typedef void * yyscan_t; +extern int ptxinfo_lex_init(yyscan_t* scanner); +extern void ptxinfo_set_in (FILE * _in_str ,yyscan_t yyscanner ); +extern int ptxinfo_parse(yyscan_t scanner, ptxinfo_data* ptxinfo); extern int ptxinfo_lex_destroy(yyscan_t scanner); static bool g_save_embedded_ptx; static int g_occupancy_sm_number; -bool ptxinfo_data::keep_intermediate_files() { - return g_keep_intermediate_files; +bool ptxinfo_data::keep_intermediate_files() {return g_keep_intermediate_files;} + +void gpgpu_context::ptx_reg_options(option_parser_t opp) +{ + option_parser_register(opp, "-save_embedded_ptx", OPT_BOOL, &g_save_embedded_ptx, + "saves ptx files embedded in binary as .ptx", + "0"); + option_parser_register(opp, "-keep", OPT_BOOL, &(ptxinfo->g_keep_intermediate_files), + "keep intermediate files created by GPGPU-Sim when interfacing with external programs", + "0"); + option_parser_register(opp, "-gpgpu_ptx_save_converted_ptxplus", OPT_BOOL, + &(ptxinfo->m_ptx_save_converted_ptxplus), + "Saved converted ptxplus to a file", + "0"); + option_parser_register(opp, "-gpgpu_occupancy_sm_number", OPT_INT32, &g_occupancy_sm_number, + "The SM number to pass to ptxas when getting register usage for computing GPU occupancy. " + "This parameter is required in the config.", + "0"); } -void gpgpu_context::ptx_reg_options(option_parser_t opp) { - option_parser_register(opp, "-save_embedded_ptx", OPT_BOOL, - &g_save_embedded_ptx, - "saves ptx files embedded in binary as .ptx", "0"); - option_parser_register(opp, "-keep", OPT_BOOL, - &(ptxinfo->g_keep_intermediate_files), - "keep intermediate files created by GPGPU-Sim when " - "interfacing with external programs", - "0"); - option_parser_register(opp, "-gpgpu_ptx_save_converted_ptxplus", OPT_BOOL, - &(ptxinfo->m_ptx_save_converted_ptxplus), - "Saved converted ptxplus to a file", "0"); - option_parser_register(opp, "-gpgpu_occupancy_sm_number", OPT_INT32, - &g_occupancy_sm_number, - "The SM number to pass to ptxas when getting register " - "usage for computing GPU occupancy. " - "This parameter is required in the config.", - "0"); +void gpgpu_context::print_ptx_file( const char *p, unsigned source_num, const char *filename ) +{ + printf("\nGPGPU-Sim PTX: file _%u.ptx contents:\n\n", source_num ); + char *s = strdup(p); + char *t = s; + unsigned n=1; + while ( *t != '\0' ) { + char *u = t; + while ( (*u != '\n') && (*u != '\0') ) u++; + unsigned last = (*u == '\0'); + *u = '\0'; + const ptx_instruction *pI = ptx_parser->ptx_instruction_lookup(filename,n); + char pc[64]; + if( pI && pI->get_PC() ) + snprintf(pc,64,"%4u", pI->get_PC() ); + else + snprintf(pc,64," "); + printf(" _%u.ptx %4u (pc=%s): %s\n", source_num, n, pc, t ); + if ( last ) break; + t = u+1; + n++; + } + free(s); + fflush(stdout); } -void gpgpu_context::print_ptx_file(const char *p, unsigned source_num, - const char *filename) { - printf("\nGPGPU-Sim PTX: file _%u.ptx contents:\n\n", source_num); - char *s = strdup(p); - char *t = s; - unsigned n = 1; - while (*t != '\0') { - char *u = t; - while ((*u != '\n') && (*u != '\0')) u++; - unsigned last = (*u == '\0'); - *u = '\0'; - const ptx_instruction *pI = ptx_parser->ptx_instruction_lookup(filename, n); - char pc[64]; - if (pI && pI->get_PC()) - snprintf(pc, 64, "%4u", pI->get_PC()); - else - snprintf(pc, 64, " "); - printf(" _%u.ptx %4u (pc=%s): %s\n", source_num, n, pc, t); - if (last) break; - t = u + 1; - n++; - } - free(s); - fflush(stdout); +char* ptxinfo_data::gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptxfilename, const std::string elffilename, const std::string sassfilename) +{ + + printf("GPGPU-Sim PTX: converting EMBEDDED .ptx file to ptxplus \n"); + + char fname_ptxplus[1024]; + snprintf(fname_ptxplus,1024,"_ptxplus_XXXXXX"); + int fd4=mkstemp(fname_ptxplus); + close(fd4); + + // Run cuobjdump_to_ptxplus + char commandline[1024]; + int result; + snprintf(commandline, 1024, "$GPGPUSIM_ROOT/build/$GPGPUSIM_CONFIG/cuobjdump_to_ptxplus/cuobjdump_to_ptxplus %s %s %s %s", + ptxfilename.c_str(), + sassfilename.c_str(), + elffilename.c_str(), + fname_ptxplus); + fflush(stdout); + printf("GPGPU-Sim PTX: calling cuobjdump_to_ptxplus\ncommandline: %s\n", commandline); + result = system(commandline); + if(result){fprintf(stderr, "GPGPU-Sim PTX: ERROR ** could not execute %s\n", commandline); exit(1);} + + + // Get ptxplus from file + std::ifstream fileStream(fname_ptxplus, std::ios::in); + std::string text, line; + while(getline(fileStream,line)) { + text += (line + "\n"); + } + fileStream.close(); + + char* ptxplus_str = new char [strlen(text.c_str())+1]; + strcpy(ptxplus_str, text.c_str()); + + if (!m_ptx_save_converted_ptxplus){ + char rm_commandline[1024]; + + snprintf(rm_commandline,1024,"rm -f %s", fname_ptxplus); + + printf("GPGPU-Sim PTX: removing temporary files using \"%s\"\n", rm_commandline); + int rm_result = system(rm_commandline); + if( rm_result != 0 ) { + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", rm_result); + exit(1); + } + } + printf("GPGPU-Sim PTX: DONE converting EMBEDDED .ptx file to ptxplus \n"); + + return ptxplus_str; } -char *ptxinfo_data::gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus( - const std::string ptxfilename, const std::string elffilename, - const std::string sassfilename) { - printf("GPGPU-Sim PTX: converting EMBEDDED .ptx file to ptxplus \n"); - - char fname_ptxplus[1024]; - snprintf(fname_ptxplus, 1024, "_ptxplus_XXXXXX"); - int fd4 = mkstemp(fname_ptxplus); - close(fd4); - - // Run cuobjdump_to_ptxplus - char commandline[1024]; - int result; - snprintf(commandline, 1024, - "$GPGPUSIM_ROOT/build/$GPGPUSIM_CONFIG/cuobjdump_to_ptxplus/" - "cuobjdump_to_ptxplus %s %s %s %s", - ptxfilename.c_str(), sassfilename.c_str(), elffilename.c_str(), - fname_ptxplus); - fflush(stdout); - printf("GPGPU-Sim PTX: calling cuobjdump_to_ptxplus\ncommandline: %s\n", - commandline); - result = system(commandline); - if (result) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** could not execute %s\n", - commandline); - exit(1); - } - - // Get ptxplus from file - std::ifstream fileStream(fname_ptxplus, std::ios::in); - std::string text, line; - while (getline(fileStream, line)) { - text += (line + "\n"); - } - fileStream.close(); - - char *ptxplus_str = new char[strlen(text.c_str()) + 1]; - strcpy(ptxplus_str, text.c_str()); - - if (!m_ptx_save_converted_ptxplus) { - char rm_commandline[1024]; - - snprintf(rm_commandline, 1024, "rm -f %s", fname_ptxplus); - - printf("GPGPU-Sim PTX: removing temporary files using \"%s\"\n", - rm_commandline); - int rm_result = system(rm_commandline); - if (rm_result != 0) { - fprintf(stderr, - "GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", - rm_result); - exit(1); + +symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_string( const char *p, unsigned source_num ) +{ + char buf[1024]; + snprintf(buf,1024,"_%u.ptx", source_num ); + if( g_save_embedded_ptx ) { + FILE *fp = fopen(buf,"w"); + fprintf(fp,"%s",p); + fclose(fp); } - } - printf("GPGPU-Sim PTX: DONE converting EMBEDDED .ptx file to ptxplus \n"); + symbol_table *symtab=init_parser(buf); + ptx_lex_init(&(ptx_parser->scanner)); + ptx__scan_string(p, ptx_parser->scanner); + int errors = ptx_parse (ptx_parser->scanner, ptx_parser); + if ( errors ) { + char fname[1024]; + snprintf(fname,1024,"_ptx_errors_XXXXXX"); + int fd=mkstemp(fname); + close(fd); + printf("GPGPU-Sim PTX: parser error detected, exiting... but first extracting .ptx to \"%s\"\n", fname); + FILE *ptxfile = fopen(fname,"w"); + fprintf(ptxfile,"%s", p ); + fclose(ptxfile); + abort(); + exit(40); + } + ptx_lex_destroy(ptx_parser->scanner); - return ptxplus_str; -} + if ( g_debug_execution >= 100 ) + print_ptx_file(p,source_num,buf); -symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_string( - const char *p, unsigned source_num) { - char buf[1024]; - snprintf(buf, 1024, "_%u.ptx", source_num); - if (g_save_embedded_ptx) { - FILE *fp = fopen(buf, "w"); - fprintf(fp, "%s", p); - fclose(fp); - } - symbol_table *symtab = init_parser(buf); - ptx_lex_init(&(ptx_parser->scanner)); - ptx__scan_string(p, ptx_parser->scanner); - int errors = ptx_parse(ptx_parser->scanner, ptx_parser); - if (errors) { - char fname[1024]; - snprintf(fname, 1024, "_ptx_errors_XXXXXX"); - int fd = mkstemp(fname); - close(fd); - printf( - "GPGPU-Sim PTX: parser error detected, exiting... but first extracting " - ".ptx to \"%s\"\n", - fname); - FILE *ptxfile = fopen(fname, "w"); - fprintf(ptxfile, "%s", p); - fclose(ptxfile); - abort(); - exit(40); - } - ptx_lex_destroy(ptx_parser->scanner); - - if (g_debug_execution >= 100) print_ptx_file(p, source_num, buf); - - printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n", buf); - return symtab; + printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n",buf); + return symtab; } -symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_filename( - const char *filename) { - symbol_table *symtab = init_parser(filename); - printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n", filename); - return symtab; +symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_filename( const char *filename ) +{ + symbol_table *symtab=init_parser(filename); + printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n",filename); + return symtab; } void fix_duplicate_errors(char fname2[1024]) { - char tempfile[1024] = "_temp_ptx"; - char commandline[1024]; - - // change the name of the ptx file to _temp_ptx - snprintf(commandline, 1024, "mv %s %s", fname2, tempfile); - printf("Running: %s\n", commandline); - int result = system(commandline); - if (result != 0) { - fprintf(stderr, - "GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", - fname2, tempfile); - exit(1); - } - - // store all of the ptx into a char array - FILE *ptxsource = fopen(tempfile, "r"); - fseek(ptxsource, 0, SEEK_END); - long filesize = ftell(ptxsource); - rewind(ptxsource); - char *ptxdata = (char *)malloc((filesize + 1) * sizeof(char)); - // Fail if we do not read the file - assert(fread(ptxdata, filesize, 1, ptxsource) == 1); - fclose(ptxsource); - - FILE *ptxdest = fopen(fname2, "w"); - std::map duplicate = get_duplicate(); - unsigned offset; - unsigned oldlinenum = 1; - unsigned linenum; - char *startptr = ptxdata; - char *funcptr; - char *tempptr = ptxdata - 1; - char *lineptr = ptxdata - 1; - - // recreate the ptx file without duplications - for (std::map::iterator iter = duplicate.begin(); - iter != duplicate.end(); iter++) { - // find the line of the next error - linenum = iter->first; - for (int i = oldlinenum; i < linenum; i++) { - lineptr = strchr(lineptr + 1, '\n'); - } - - // find the end of the current section to be copied over - // then find the start of the next section that will be copied - if (strcmp("function", iter->second) == 0) { - // get location of most recent .func - while (tempptr < lineptr && tempptr != NULL) { - funcptr = tempptr; - tempptr = strstr(funcptr + 1, ".func"); - } - - // get the start of the previous line - offset = 0; - while (*(funcptr - offset) != '\n') offset++; - - fwrite(startptr, sizeof(char), funcptr - offset + 1 - startptr, ptxdest); - - // find next location of startptr - if (*(lineptr + 3) == ';') { - // for function definitions - startptr = lineptr + 5; - } else if (*(lineptr + 3) == '{') { - // for functions enclosed with curly brackets - offset = 5; - unsigned bracket = 1; - while (bracket != 0) { - if (*(lineptr + offset) == '{') - bracket++; - else if (*(lineptr + offset) == '}') - bracket--; - offset++; - } - startptr = lineptr + offset + 1; - } else { - printf("GPGPU-Sim PTX: ERROR ** Unrecognized function format\n"); - abort(); - } - } else if (strcmp("variable", iter->second) == 0) { - fwrite(startptr, sizeof(char), (int)(lineptr + 1 - startptr), ptxdest); - - // find next location of startptr - offset = 1; - while (*(lineptr + offset) != '\n') offset++; - startptr = lineptr + offset + 1; - } else { - printf("GPGPU-Sim PTX: ERROR ** Unsupported duplicate type: %s\n", - iter->second); - } - - oldlinenum = linenum; - } - // copy over the rest of the file - fwrite(startptr, sizeof(char), ptxdata + filesize - startptr, ptxdest); - - // cleanup - free(ptxdata); - fclose(ptxdest); - snprintf(commandline, 1024, "rm -f %s", tempfile); - printf("Running: %s\n", commandline); - result = system(commandline); - if (result != 0) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile); - exit(1); - } + char tempfile[1024] = "_temp_ptx"; + char commandline[1024]; + + // change the name of the ptx file to _temp_ptx + snprintf(commandline,1024,"mv %s %s",fname2,tempfile); + printf("Running: %s\n", commandline); + int result = system(commandline); + if (result != 0) { + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", fname2, tempfile); + exit(1); + } + + // store all of the ptx into a char array + FILE *ptxsource = fopen(tempfile,"r"); + fseek(ptxsource, 0, SEEK_END); + long filesize = ftell(ptxsource); + rewind(ptxsource); + char *ptxdata = (char*)malloc((filesize+1)*sizeof(char)); + // Fail if we do not read the file + assert(fread(ptxdata, filesize, 1, ptxsource) == 1); + fclose(ptxsource); + + FILE *ptxdest = fopen(fname2,"w"); + std::map duplicate = get_duplicate(); + unsigned offset; + unsigned oldlinenum = 1; + unsigned linenum; + char *startptr = ptxdata; + char *funcptr; + char *tempptr = ptxdata - 1; + char *lineptr = ptxdata - 1; + + // recreate the ptx file without duplications + for ( std::map::iterator iter = duplicate.begin(); + iter != duplicate.end(); + iter++){ + // find the line of the next error + linenum = iter->first; + for (int i = oldlinenum; i < linenum; i++) { + lineptr = strchr(lineptr + 1, '\n'); + } + + // find the end of the current section to be copied over + // then find the start of the next section that will be copied + if (strcmp("function", iter->second) == 0) { + // get location of most recent .func + while (tempptr < lineptr && tempptr != NULL) { + funcptr = tempptr; + tempptr = strstr(funcptr + 1, ".func"); + } + + // get the start of the previous line + offset = 0; + while (*(funcptr - offset) != '\n') offset++; + + fwrite(startptr, sizeof(char), funcptr - offset + 1 - startptr, ptxdest); + + //find next location of startptr + if (*(lineptr + 3) == ';') { + // for function definitions + startptr = lineptr + 5; + } else if (*(lineptr + 3) == '{') { + // for functions enclosed with curly brackets + offset = 5; + unsigned bracket = 1; + while (bracket != 0) { + if (*(lineptr + offset) == '{') bracket++; + else if (*(lineptr + offset) == '}') bracket--; + offset++; + } + startptr = lineptr + offset + 1; + } else { + printf("GPGPU-Sim PTX: ERROR ** Unrecognized function format\n"); + abort(); + } + } else if (strcmp("variable", iter->second) == 0) { + fwrite(startptr, sizeof(char), (int)(lineptr + 1 - startptr), ptxdest); + + //find next location of startptr + offset = 1; + while (*(lineptr + offset) != '\n') offset++; + startptr = lineptr + offset + 1; + } else { + printf("GPGPU-Sim PTX: ERROR ** Unsupported duplicate type: %s\n", iter->second); + } + + oldlinenum = linenum; + } + // copy over the rest of the file + fwrite(startptr, sizeof(char), ptxdata + filesize - startptr, ptxdest); + + // cleanup + free(ptxdata); + fclose(ptxdest); + snprintf(commandline,1024,"rm -f %s",tempfile); + printf("Running: %s\n", commandline); + result = system(commandline); + if (result != 0) { + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile); + exit(1); + } } -// we need the application name here too. -char *get_app_binary_name() { - char exe_path[1025]; - char *self_exe_path; + +//we need the application name here too. +char* get_app_binary_name(){ + char exe_path[1025]; + char *self_exe_path; #ifdef __APPLE__ - // AMRUTH: get apple device and check the result. - printf("WARNING: not tested for Apple-mac devices \n"); - abort(); + //AMRUTH: get apple device and check the result. + printf("WARNING: not tested for Apple-mac devices \n"); + abort(); #else - std::stringstream exec_link; - exec_link << "/proc/self/exe"; - ssize_t path_length = readlink(exec_link.str().c_str(), exe_path, 1024); - assert(path_length != -1); - exe_path[path_length] = '\0'; - - char *token = strtok(exe_path, "/"); - while (token != NULL) { - self_exe_path = token; - token = strtok(NULL, "/"); - } + std::stringstream exec_link; + exec_link << "/proc/self/exe"; + ssize_t path_length = readlink(exec_link.str().c_str(), exe_path, 1024); + assert(path_length != -1); + exe_path[path_length] = '\0'; + + char *token = strtok(exe_path, "/"); + while(token !=NULL){ + self_exe_path = token; + token = strtok(NULL,"/"); + } #endif - self_exe_path = strtok(self_exe_path, "."); - printf("self exe links to: %s\n", self_exe_path); - return self_exe_path; + self_exe_path = strtok(self_exe_path, "."); + printf("self exe links to: %s\n", self_exe_path); + return self_exe_path; } -void gpgpu_context::gpgpu_ptx_info_load_from_filename(const char *filename, - unsigned sm_version) { - std::string ptxas_filename(std::string(filename) + "as"); - char buff[1024], extra_flags[1024]; - extra_flags[0] = 0; - if (!device_runtime->g_cdp_enabled) - snprintf(extra_flags, 1024, "--gpu-name=sm_%u", sm_version); - else - snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", sm_version); - snprintf( - buff, 1024, - "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", - extra_flags, filename, ptxas_filename.c_str()); - int result = system(buff); - if (result != 0) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); - } - - FILE *ptxinfo_in; - ptxinfo->g_ptxinfo_filename = strdup(ptxas_filename.c_str()); - ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename, "r"); - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); +void gpgpu_context::gpgpu_ptx_info_load_from_filename( const char *filename, unsigned sm_version) +{ + std::string ptxas_filename(std::string(filename) + "as"); + char buff[1024], extra_flags[1024]; + extra_flags[0]=0; + if(!device_runtime->g_cdp_enabled) + snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version); + else + snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version); + snprintf(buff,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, filename, ptxas_filename.c_str()); + int result = system(buff); + if( result != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } + + FILE *ptxinfo_in; + ptxinfo->g_ptxinfo_filename = strdup(ptxas_filename.c_str()); + ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename,"r"); + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); } -void gpgpu_context::gpgpu_ptxinfo_load_from_string(const char *p_for_info, - unsigned source_num, - unsigned sm_version, - int no_of_ptx) { - // do ptxas for individual files instead of one big embedded ptx. This - // prevents the duplicate defs and declarations. - char ptx_file[1000]; - char *name = get_app_binary_name(); - char commandline[4096], fname[1024], fname2[1024], - final_tempfile_ptxinfo[1024], tempfile_ptxinfo[1024]; - for (int index = 1; index <= no_of_ptx; index++) { - snprintf(ptx_file, 1000, "%s.%d.sm_%u.ptx", name, index, sm_version); - snprintf(fname, 1024, "_ptx_XXXXXX"); - int fd = mkstemp(fname); - close(fd); - - printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", - fname); - snprintf(commandline, 4096, "cat %s > %s", ptx_file, fname); - if (system(commandline) != 0) { - printf("ERROR: %s command failed\n", commandline); - exit(0); - } - - snprintf(fname2, 1024, "_ptx2_XXXXXX"); - fd = mkstemp(fname2); - close(fd); - char commandline2[4096]; - snprintf(commandline2, 4096, - "cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, " - "texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 " - "\\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed " - "'s/const\\[.\\]/const\\[0\\]/g' > %s", - fname, fname2); - printf("Running: %s\n", commandline2); - int result = system(commandline2); - if (result != 0) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); - printf( - " Ensure you have write access to simulation " - "directory\n"); - printf(" and have \'cat\' and \'sed\' in your path.\n"); - exit(1); - } - - snprintf(tempfile_ptxinfo, 1024, "%sinfo", fname); - char extra_flags[1024]; - extra_flags[0] = 0; +void gpgpu_context::gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num, unsigned sm_version, int no_of_ptx ) +{ + //do ptxas for individual files instead of one big embedded ptx. This prevents the duplicate defs and declarations. + char ptx_file[1000]; + char *name=get_app_binary_name(); + char commandline[4096], fname[1024], fname2[1024], final_tempfile_ptxinfo[1024], tempfile_ptxinfo[1024]; + for (int index=1; index <= no_of_ptx; index++){ + snprintf(ptx_file, 1000, "%s.%d.sm_%u.ptx", name, index, sm_version); + snprintf(fname,1024,"_ptx_XXXXXX"); + int fd=mkstemp(fname); + close(fd); + + printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname); + snprintf(commandline,4096,"cat %s > %s",ptx_file, fname); + if (system(commandline) !=0) { + printf("ERROR: %s command failed\n", commandline); + exit(0); + } + + snprintf(fname2,1024,"_ptx2_XXXXXX"); + fd=mkstemp(fname2); + close(fd); + char commandline2[4096]; + snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2); + printf("Running: %s\n", commandline2); + int result = system(commandline2); + if( result != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); + printf(" Ensure you have write access to simulation directory\n"); + printf(" and have \'cat\' and \'sed\' in your path.\n"); + exit(1); + } + + snprintf(tempfile_ptxinfo,1024,"%sinfo",fname); + char extra_flags[1024]; + extra_flags[0]=0; #if CUDART_VERSION >= 3000 - if (g_occupancy_sm_number == 0) { - fprintf( - stderr, - "gpgpusim.config must specify the sm version for the GPU that you " - "use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n" - "The register file size is specifically tied to the sm version used " - "to querry ptxas for register usage.\n" - "A register size/SM mismatch may result in occupancy differences."); - exit(1); + if ( g_occupancy_sm_number == 0 ) { + fprintf( stderr, "gpgpusim.config must specify the sm version for the GPU that you use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n" + "The register file size is specifically tied to the sm version used to querry ptxas for register usage.\n" + "A register size/SM mismatch may result in occupancy differences." ); + exit(1); } - if (!device_runtime->g_cdp_enabled) - snprintf(extra_flags, 1024, "--gpu-name=sm_%u", g_occupancy_sm_number); + if(!device_runtime->g_cdp_enabled) + snprintf(extra_flags,1024,"--gpu-name=sm_%u", g_occupancy_sm_number); else - snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", - g_occupancy_sm_number); + snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",g_occupancy_sm_number); #endif - snprintf(commandline, 1024, - "$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file " - "/dev/null 2> %s", + snprintf(commandline,1024,"$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", extra_flags, fname2, tempfile_ptxinfo); printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); result = system(commandline); - if (result != 0) { - // 65280 = duplicate errors - if (result == 65280) { - FILE *ptxinfo_in; - ptxinfo_in = fopen(tempfile_ptxinfo, "r"); - ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); - - fix_duplicate_errors(fname2); - snprintf(commandline, 1024, - "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file " - "/dev/null 2> %s", - extra_flags, fname2, tempfile_ptxinfo); - printf("GPGPU-Sim PTX: regenerating ptxinfo using \"%s\"\n", - commandline); - result = system(commandline); - } - if (result != 0) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); - } + if( result != 0 ) { + // 65280 = duplicate errors + if (result == 65280) { + FILE *ptxinfo_in; + ptxinfo_in = fopen(tempfile_ptxinfo,"r"); + ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); + + fix_duplicate_errors(fname2); + snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: regenerating ptxinfo using \"%s\"\n", commandline); + result = system(commandline); + } + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } + } } - } - - // TODO: duplicate code! move it into a function so that it can be reused! - if (no_of_ptx == 0) { - // For CDP, we dump everything. So no_of_ptx will be 0. - snprintf(fname, 1024, "_ptx_XXXXXX"); - int fd = mkstemp(fname); - close(fd); - - printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", - fname); - FILE *ptxfile = fopen(fname, "w"); - fprintf(ptxfile, "%s", p_for_info); - fclose(ptxfile); - - snprintf(fname2, 1024, "_ptx2_XXXXXX"); - fd = mkstemp(fname2); - close(fd); - char commandline2[4096]; - snprintf(commandline2, 4096, - "cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, " - "texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 " - "\\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed " - "'s/const\\[.\\]/const\\[0\\]/g' > %s", - fname, fname2); - printf("Running: %s\n", commandline2); - int result = system(commandline2); - if (result != 0) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); - printf( - " Ensure you have write access to simulation " - "directory\n"); - printf(" and have \'cat\' and \'sed\' in your path.\n"); - exit(1); + + //TODO: duplicate code! move it into a function so that it can be reused! + if(no_of_ptx==0) { + //For CDP, we dump everything. So no_of_ptx will be 0. + snprintf(fname,1024,"_ptx_XXXXXX"); + int fd=mkstemp(fname); + close(fd); + + printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname); + FILE *ptxfile = fopen(fname,"w"); + fprintf(ptxfile,"%s", p_for_info); + fclose(ptxfile); + + snprintf(fname2,1024,"_ptx2_XXXXXX"); + fd=mkstemp(fname2); + close(fd); + char commandline2[4096]; + snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2); + printf("Running: %s\n", commandline2); + int result = system(commandline2); + if( result != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); + printf(" Ensure you have write access to simulation directory\n"); + printf(" and have \'cat\' and \'sed\' in your path.\n"); + exit(1); + } + //char tempfile_ptxinfo[1024]; + snprintf(tempfile_ptxinfo,1024,"%sinfo",fname); + char extra_flags[1024]; + extra_flags[0]=0; + + #if CUDART_VERSION >= 3000 + if (sm_version == 0) sm_version = 20; + if(!device_runtime->g_cdp_enabled) + snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version); + else + snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version); + #endif + + snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); + fflush(stdout); + result = system(commandline); + if( result != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } } - // char tempfile_ptxinfo[1024]; - snprintf(tempfile_ptxinfo, 1024, "%sinfo", fname); - char extra_flags[1024]; - extra_flags[0] = 0; -#if CUDART_VERSION >= 3000 - if (sm_version == 0) sm_version = 20; - if (!device_runtime->g_cdp_enabled) - snprintf(extra_flags, 1024, "--gpu-name=sm_%u", sm_version); - else - snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", - sm_version); -#endif + //Now that we got resource usage per kernel in a ptx file, we dump all into one file and pass it to rest of the code as usual. + if(no_of_ptx>0){ + char commandline3[4096]; + snprintf(final_tempfile_ptxinfo,1024,"f_tempfile_ptx"); + snprintf(commandline3,4096, "cat *info > %s", final_tempfile_ptxinfo); + if (system(commandline3)!=0) { + printf("ERROR: Either we dont have info files or cat is not working \n"); + printf("ERROR: %s command failed\n",commandline3); + exit(1); + } + } - snprintf( - commandline, 1024, - "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", - extra_flags, fname2, tempfile_ptxinfo); - printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); - fflush(stdout); - result = system(commandline); - if (result != 0) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); - } - } - - // Now that we got resource usage per kernel in a ptx file, we dump all into - // one file and pass it to rest of the code as usual. - if (no_of_ptx > 0) { - char commandline3[4096]; - snprintf(final_tempfile_ptxinfo, 1024, "f_tempfile_ptx"); - snprintf(commandline3, 4096, "cat *info > %s", final_tempfile_ptxinfo); - if (system(commandline3) != 0) { - printf("ERROR: Either we dont have info files or cat is not working \n"); - printf("ERROR: %s command failed\n", commandline3); - exit(1); - } - } - - if (no_of_ptx > 0) - ptxinfo->g_ptxinfo_filename = final_tempfile_ptxinfo; - else - ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; - FILE *ptxinfo_in; - ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename, "r"); - - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); - - snprintf(commandline, 1024, "rm -f *info"); - if (system(commandline) != 0) { - printf("GPGPU-Sim PTX: ERROR ** while removing temporary info files\n"); - exit(1); - } - if (!g_save_embedded_ptx) { - if (no_of_ptx > 0) - snprintf(commandline, 1024, "rm -f %s %s %s", fname, fname2, - final_tempfile_ptxinfo); + if(no_of_ptx>0) + ptxinfo->g_ptxinfo_filename = final_tempfile_ptxinfo; else - snprintf(commandline, 1024, "rm -f %s %s %s", fname, fname2, - tempfile_ptxinfo); - printf("GPGPU-Sim PTX: removing ptxinfo using \"%s\"\n", commandline); - if (system(commandline) != 0) { - printf("GPGPU-Sim PTX: ERROR ** while removing temporary files\n"); - exit(1); + ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; + FILE *ptxinfo_in; + ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename,"r"); + + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); + + snprintf(commandline,1024,"rm -f *info"); + if( system(commandline) != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while removing temporary info files\n"); + exit(1); + } + if( ! g_save_embedded_ptx ) { + if(no_of_ptx>0) + snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, final_tempfile_ptxinfo); + else + snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: removing ptxinfo using \"%s\"\n", commandline); + if( system(commandline) != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while removing temporary files\n"); + exit(1); + } } - } } diff --git a/src/cuda-sim/ptx_loader.h b/src/cuda-sim/ptx_loader.h index 17c4524..d8f1cbc 100644 --- a/src/cuda-sim/ptx_loader.h +++ b/src/cuda-sim/ptx_loader.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -33,22 +31,22 @@ #define PTXINFO_LINEBUF_SIZE 1024 class gpgpu_context; -typedef void* yyscan_t; -class ptxinfo_data { - public: - ptxinfo_data(gpgpu_context* ctx) { gpgpu_ctx = ctx; } - yyscan_t scanner; - char linebuf[PTXINFO_LINEBUF_SIZE]; - unsigned col; - const char* g_ptxinfo_filename; - class gpgpu_context* gpgpu_ctx; - bool g_keep_intermediate_files; - bool m_ptx_save_converted_ptxplus; - void ptxinfo_addinfo(); - bool keep_intermediate_files(); - char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus( - const std::string ptx_str, const std::string sass_str, - const std::string elf_str); +typedef void * yyscan_t; +class ptxinfo_data{ + public: + ptxinfo_data(gpgpu_context* ctx) { + gpgpu_ctx = ctx; + } + yyscan_t scanner; + char linebuf[PTXINFO_LINEBUF_SIZE]; + unsigned col; + const char *g_ptxinfo_filename; + class gpgpu_context* gpgpu_ctx; + bool g_keep_intermediate_files; + bool m_ptx_save_converted_ptxplus; + void ptxinfo_addinfo(); + bool keep_intermediate_files(); + char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptx_str, const std::string sass_str, const std::string elf_str); }; #endif diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index c135a6c..a4f4a0c 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -28,971 +26,998 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "ptx_parser.h" -#include "../../libcuda/gpgpu_context.h" #include "ptx_ir.h" +#include "../../libcuda/gpgpu_context.h" -typedef void *yyscan_t; -#include +typedef void * yyscan_t; #include "ptx.tab.h" +#include -extern int ptx_get_lineno(yyscan_t yyscanner); -extern YYSTYPE *ptx_get_lval(yyscan_t yyscanner); -extern int ptx_error(yyscan_t yyscanner, const char *s); -extern int ptx_lex_init(yyscan_t *scanner); -extern void ptx_set_in(FILE *_in_str, yyscan_t yyscanner); -extern FILE *ptx_get_in(yyscan_t yyscanner); -extern int ptx_parse(yyscan_t scanner, ptx_recognizer *recognizer); +extern int ptx_get_lineno (yyscan_t yyscanner ); +extern YYSTYPE* ptx_get_lval (yyscan_t yyscanner ); +extern int ptx_error( yyscan_t yyscanner, const char *s ); +extern int ptx_lex_init(yyscan_t* scanner); +extern void ptx_set_in(FILE * _in_str ,yyscan_t yyscanner ); +extern FILE *ptx_get_in (yyscan_t yyscanner ); +extern int ptx_parse(yyscan_t scanner, ptx_recognizer* recognizer); extern int ptx_lex_destroy(yyscan_t scanner); -void ptx_recognizer::set_ptx_warp_size(const struct core_config *warp_size) { - g_shader_core_config = warp_size; -} - -#define PTX_PARSE_DPRINTF(...) \ - if (g_debug_ir_generation) { \ - printf(" %s:%u => ", gpgpu_ctx->g_filename, ptx_get_lineno(scanner)); \ - printf(" (%s:%u) ", __FILE__, __LINE__); \ - printf(__VA_ARGS__); \ - printf("\n"); \ - fflush(stdout); \ - } - -static std::map g_ptx_token_decode; - -const char *decode_token(int type) { return g_ptx_token_decode[type].c_str(); } - -void ptx_recognizer::read_parser_environment_variables() { - gpgpu_ctx->g_filename = getenv("PTX_SIM_KERNELFILE"); - char *dbg_level = getenv("PTX_SIM_DEBUG"); - if (dbg_level && strlen(dbg_level)) { - int debug_execution = 0; - sscanf(dbg_level, "%d", &debug_execution); - if (debug_execution >= 30) g_debug_ir_generation = true; - } -} - -void ptx_recognizer::init_directive_state() { - PTX_PARSE_DPRINTF("init_directive_state"); - g_space_spec = undefined_space; - g_ptr_spec = undefined_space; - g_scalar_type_spec = -1; - g_vector_spec = -1; - g_opcode = -1; - g_alignment_spec = -1; - g_size = -1; - g_extern_spec = 0; - g_scalar_type.clear(); - g_operands.clear(); - g_last_symbol = NULL; -} - -void ptx_recognizer::init_instruction_state() { - PTX_PARSE_DPRINTF("init_instruction_state"); - g_pred = NULL; - g_neg_pred = 0; - g_pred_mod = -1; - g_label = NULL; - g_opcode = -1; - g_options.clear(); - g_wmma_options.clear(); - g_return_var = operand_info(gpgpu_ctx); - init_directive_state(); -} - -symbol_table *gpgpu_context::init_parser(const char *ptx_filename) { - g_filename = strdup(ptx_filename); - if (g_global_allfiles_symbol_table == NULL) { - g_global_allfiles_symbol_table = - new symbol_table("global_allfiles", 0, NULL, this); - ptx_parser->g_global_symbol_table = ptx_parser->g_current_symbol_table = - g_global_allfiles_symbol_table; - } -/*else { - g_global_symbol_table = g_current_symbol_table = new -symbol_table("global",0,g_global_allfiles_symbol_table); -}*/ - -#define DEF(X, Y) g_ptx_token_decode[X] = Y; +void ptx_recognizer::set_ptx_warp_size(const struct core_config * warp_size) +{ + g_shader_core_config=warp_size; +} + + +#define PTX_PARSE_DPRINTF(...) \ + if( g_debug_ir_generation ) { \ + printf(" %s:%u => ",gpgpu_ctx->g_filename,ptx_get_lineno(scanner)); \ + printf(" (%s:%u) ", __FILE__, __LINE__); \ + printf(__VA_ARGS__); \ + printf("\n"); \ + fflush(stdout); \ + } + +static std::map g_ptx_token_decode; + +const char *decode_token( int type ) +{ + return g_ptx_token_decode[type].c_str(); +} + +void ptx_recognizer::read_parser_environment_variables() +{ + gpgpu_ctx->g_filename = getenv("PTX_SIM_KERNELFILE"); + char *dbg_level = getenv("PTX_SIM_DEBUG"); + if ( dbg_level && strlen(dbg_level) ) { + int debug_execution=0; + sscanf(dbg_level,"%d", &debug_execution); + if ( debug_execution >= 30 ) + g_debug_ir_generation=true; + } +} + +void ptx_recognizer::init_directive_state() +{ + PTX_PARSE_DPRINTF("init_directive_state"); + g_space_spec=undefined_space; + g_ptr_spec=undefined_space; + g_scalar_type_spec=-1; + g_vector_spec=-1; + g_opcode=-1; + g_alignment_spec = -1; + g_size = -1; + g_extern_spec = 0; + g_scalar_type.clear(); + g_operands.clear(); + g_last_symbol = NULL; +} + +void ptx_recognizer::init_instruction_state() +{ + PTX_PARSE_DPRINTF("init_instruction_state"); + g_pred = NULL; + g_neg_pred = 0; + g_pred_mod = -1; + g_label = NULL; + g_opcode = -1; + g_options.clear(); + g_wmma_options.clear(); + g_return_var = operand_info(gpgpu_ctx); + init_directive_state(); +} + +symbol_table * gpgpu_context::init_parser( const char *ptx_filename ) +{ + g_filename = strdup(ptx_filename); + if (g_global_allfiles_symbol_table == NULL) { + g_global_allfiles_symbol_table = new symbol_table("global_allfiles", 0, NULL, this); + ptx_parser->g_global_symbol_table = ptx_parser->g_current_symbol_table = g_global_allfiles_symbol_table; + } + /*else { + g_global_symbol_table = g_current_symbol_table = new symbol_table("global",0,g_global_allfiles_symbol_table); + }*/ + +#define DEF(X,Y) g_ptx_token_decode[X] = Y; #include "ptx_parser_decode.def" #undef DEF - g_ptx_token_decode[undefined_space] = "undefined_space"; - g_ptx_token_decode[undefined_space] = "undefined_space=0"; - g_ptx_token_decode[reg_space] = "reg_space"; - g_ptx_token_decode[local_space] = "local_space"; - g_ptx_token_decode[shared_space] = "shared_space"; - g_ptx_token_decode[param_space_unclassified] = "param_space_unclassified"; - g_ptx_token_decode[param_space_kernel] = "param_space_kernel"; - g_ptx_token_decode[param_space_local] = "param_space_local"; - g_ptx_token_decode[const_space] = "const_space"; - g_ptx_token_decode[tex_space] = "tex_space"; - g_ptx_token_decode[surf_space] = "surf_space"; - g_ptx_token_decode[global_space] = "global_space"; - g_ptx_token_decode[generic_space] = "generic_space"; - g_ptx_token_decode[instruction_space] = "instruction_space"; - - ptx_lex_init(&(ptx_parser->scanner)); - ptx_parser->init_directive_state(); - ptx_parser->init_instruction_state(); - - FILE *ptx_in; - ptx_in = fopen(ptx_filename, "r"); - ptx_set_in(ptx_in, ptx_parser->scanner); - ptx_parse(ptx_parser->scanner, ptx_parser); - ptx_in = ptx_get_in(ptx_parser->scanner); - ptx_lex_destroy(ptx_parser->scanner); - fclose(ptx_in); - return ptx_parser->g_global_symbol_table; -} - -void ptx_recognizer::start_function(int entry_point) { - PTX_PARSE_DPRINTF("start_function"); - init_directive_state(); - init_instruction_state(); - g_entry_point = entry_point; - g_func_info = NULL; - g_entry_func_param_index = 0; -} - -void ptx_recognizer::add_function_name(const char *name) { - PTX_PARSE_DPRINTF( - "add_function_name %s %s", name, - ((g_entry_point == 1) ? "(entrypoint)" - : ((g_entry_point == 2) ? "(extern)" : ""))); - bool prior_decl = g_global_symbol_table->add_function_decl( - name, g_entry_point, &g_func_info, &g_current_symbol_table); - if (g_add_identifier_cached__identifier) { - add_identifier(g_add_identifier_cached__identifier, - g_add_identifier_cached__array_dim, - g_add_identifier_cached__array_ident); - free(g_add_identifier_cached__identifier); - g_add_identifier_cached__identifier = NULL; - g_func_info->add_return_var(g_last_symbol); - init_directive_state(); - } - if (prior_decl) { - g_func_info->remove_args(); - } - g_global_symbol_table->add_function(g_func_info, gpgpu_ctx->g_filename, - ptx_get_lineno(scanner)); -} - -// Jin: handle instruction group for cdp + g_ptx_token_decode[undefined_space] = "undefined_space"; + g_ptx_token_decode[undefined_space] = "undefined_space=0"; + g_ptx_token_decode[reg_space] = "reg_space"; + g_ptx_token_decode[local_space] = "local_space"; + g_ptx_token_decode[shared_space] = "shared_space"; + g_ptx_token_decode[param_space_unclassified] = "param_space_unclassified"; + g_ptx_token_decode[param_space_kernel] = "param_space_kernel"; + g_ptx_token_decode[param_space_local] = "param_space_local"; + g_ptx_token_decode[const_space] = "const_space"; + g_ptx_token_decode[tex_space] = "tex_space"; + g_ptx_token_decode[surf_space] = "surf_space"; + g_ptx_token_decode[global_space] = "global_space"; + g_ptx_token_decode[generic_space] = "generic_space"; + g_ptx_token_decode[instruction_space] = "instruction_space"; + + ptx_lex_init(&(ptx_parser->scanner)); + ptx_parser->init_directive_state(); + ptx_parser->init_instruction_state(); + + FILE *ptx_in; + ptx_in = fopen(ptx_filename, "r"); + ptx_set_in(ptx_in, ptx_parser->scanner); + ptx_parse(ptx_parser->scanner, ptx_parser); + ptx_in = ptx_get_in(ptx_parser->scanner); + ptx_lex_destroy(ptx_parser->scanner); + fclose(ptx_in); + return ptx_parser->g_global_symbol_table; +} + + +void ptx_recognizer::start_function( int entry_point ) +{ + PTX_PARSE_DPRINTF("start_function"); + init_directive_state(); + init_instruction_state(); + g_entry_point = entry_point; + g_func_info = NULL; + g_entry_func_param_index=0; +} + +void ptx_recognizer::add_function_name( const char *name ) +{ + PTX_PARSE_DPRINTF("add_function_name %s %s", name, ((g_entry_point==1)?"(entrypoint)":((g_entry_point==2)?"(extern)":""))); + bool prior_decl = g_global_symbol_table->add_function_decl( name, g_entry_point, &g_func_info, &g_current_symbol_table ); + if( g_add_identifier_cached__identifier ) { + add_identifier( g_add_identifier_cached__identifier, + g_add_identifier_cached__array_dim, + g_add_identifier_cached__array_ident ); + free( g_add_identifier_cached__identifier ); + g_add_identifier_cached__identifier = NULL; + g_func_info->add_return_var( g_last_symbol ); + init_directive_state(); + } + if( prior_decl ) { + g_func_info->remove_args(); + } + g_global_symbol_table->add_function( g_func_info, gpgpu_ctx->g_filename, ptx_get_lineno(scanner) ); +} + +//Jin: handle instruction group for cdp void ptx_recognizer::start_inst_group() { - PTX_PARSE_DPRINTF("start_instruction_group"); - g_current_symbol_table = g_current_symbol_table->start_inst_group(); + PTX_PARSE_DPRINTF("start_instruction_group"); + g_current_symbol_table = g_current_symbol_table->start_inst_group(); } void ptx_recognizer::end_inst_group() { - PTX_PARSE_DPRINTF("end_instruction_group"); - g_current_symbol_table = g_current_symbol_table->end_inst_group(); -} - -void ptx_recognizer::add_directive() { - PTX_PARSE_DPRINTF("add_directive"); - init_directive_state(); -} - -#define mymax(a, b) ((a) > (b) ? (a) : (b)) - -void ptx_recognizer::end_function() { - PTX_PARSE_DPRINTF("end_function"); - - init_directive_state(); - init_instruction_state(); - g_max_regs_per_thread = mymax(g_max_regs_per_thread, - (g_current_symbol_table->next_reg_num() - 1)); - g_func_info->add_inst(g_instructions); - g_instructions.clear(); - gpgpu_ptx_assemble(g_func_info->get_name(), g_func_info); - g_current_symbol_table = g_global_symbol_table; - - PTX_PARSE_DPRINTF("function %s, PC = %d\n", g_func_info->get_name().c_str(), - g_func_info->get_start_PC()); -} - -#define parse_error(msg, ...) \ - parse_error_impl(__FILE__, __LINE__, msg, ##__VA_ARGS__) -#define parse_assert(cond, msg, ...) \ - parse_assert_impl((cond), __FILE__, __LINE__, msg, ##__VA_ARGS__) - -void ptx_recognizer::parse_error_impl(const char *file, unsigned line, - const char *msg, ...) { - va_list ap; - char buf[1024]; - va_start(ap, msg); - vsnprintf(buf, 1024, msg, ap); - va_end(ap); - - g_error_detected = 1; - printf("%s:%u: Parse error: %s (%s:%u)\n\n", gpgpu_ctx->g_filename, - ptx_get_lineno(scanner), buf, file, line); - ptx_error(scanner, NULL); - abort(); - exit(1); -} - -void ptx_recognizer::parse_assert_impl(int test_value, const char *file, - unsigned line, const char *msg, ...) { - va_list ap; - char buf[1024]; - va_start(ap, msg); - vsnprintf(buf, 1024, msg, ap); - va_end(ap); - - if (test_value == 0) parse_error_impl(file, line, msg); -} - -void ptx_recognizer::set_return() { - parse_assert((g_opcode == CALL_OP || g_opcode == CALLP_OP), - "only call can have return value"); - g_operands.front().set_return(); - g_return_var = g_operands.front(); -} - -const ptx_instruction *ptx_recognizer::ptx_instruction_lookup( - const char *filename, unsigned linenumber) { - std::map >::iterator - f = g_inst_lookup.find(filename); - if (f == g_inst_lookup.end()) return NULL; - std::map::iterator l = - f->second.find(linenumber); - if (l == f->second.end()) return NULL; - return l->second; -} - -void ptx_recognizer::add_instruction() { - PTX_PARSE_DPRINTF("add_instruction: %s", - ((g_opcode > 0) ? g_opcode_string[g_opcode] : "