diff options
| author | Nick <[email protected]> | 2019-09-13 07:48:07 -0400 |
|---|---|---|
| committer | Nick <[email protected]> | 2019-09-13 07:48:07 -0400 |
| commit | 5c99f0fa58caf45f4d457894413aa11c03afdb7d (patch) | |
| tree | dbdd4b0edd3e587f3ab5fc2d6a00dda4ddb7c444 /src/cuda-sim | |
| parent | ee22431b623355df3f3a4817f812f4fc64756d03 (diff) | |
Revert "Add src/cuda-sim formatting"
This reverts commit 0c023e41809dba8897c37af6bb03e5c3aa9ebc5e.
Diffstat (limited to 'src/cuda-sim')
| -rw-r--r-- | src/cuda-sim/cuda-math.h | 429 | ||||
| -rw-r--r-- | src/cuda-sim/cuda-sim.cc | 4539 | ||||
| -rw-r--r-- | src/cuda-sim/cuda-sim.h | 287 | ||||
| -rw-r--r-- | src/cuda-sim/cuda_device_printf.cc | 165 | ||||
| -rw-r--r-- | src/cuda-sim/cuda_device_printf.h | 10 | ||||
| -rw-r--r-- | src/cuda-sim/cuda_device_runtime.cc | 518 | ||||
| -rw-r--r-- | src/cuda-sim/cuda_device_runtime.h | 91 | ||||
| -rw-r--r-- | src/cuda-sim/half.h | 6061 | ||||
| -rw-r--r-- | src/cuda-sim/instructions.cc | 10108 | ||||
| -rw-r--r-- | src/cuda-sim/memory.cc | 318 | ||||
| -rw-r--r-- | src/cuda-sim/memory.h | 144 | ||||
| -rw-r--r-- | src/cuda-sim/opcodes.h | 77 | ||||
| -rw-r--r-- | src/cuda-sim/ptx-stats.cc | 366 | ||||
| -rw-r--r-- | src/cuda-sim/ptx-stats.h | 54 | ||||
| -rw-r--r-- | src/cuda-sim/ptx_ir.cc | 2451 | ||||
| -rw-r--r-- | src/cuda-sim/ptx_ir.h | 2755 | ||||
| -rw-r--r-- | src/cuda-sim/ptx_loader.cc | 895 | ||||
| -rw-r--r-- | src/cuda-sim/ptx_loader.h | 38 | ||||
| -rw-r--r-- | src/cuda-sim/ptx_parser.cc | 1549 | ||||
| -rw-r--r-- | src/cuda-sim/ptx_parser.h | 302 | ||||
| -rw-r--r-- | src/cuda-sim/ptx_sim.cc | 955 | ||||
| -rw-r--r-- | src/cuda-sim/ptx_sim.h | 856 |
22 files changed, 15453 insertions, 17515 deletions
diff --git a/src/cuda-sim/cuda-math.h b/src/cuda-sim/cuda-math.h index 97ce130..9a5468c 100644 --- a/src/cuda-sim/cuda-math.h +++ b/src/cuda-sim/cuda-math.h @@ -1,6 +1,6 @@ // This file created from vector_types.h distributed with CUDA 1.1 // (see original copyright notice below) -// +// // Changes Copyright (c) 2009-2011, Tor M. Aamodt, Wilson W.L. Fung // The University of British Columbia // All rights reserved. @@ -10,16 +10,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -30,41 +28,43 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + /* * Copyright 1993-2007 NVIDIA Corporation. All rights reserved. * - * NOTICE TO USER: + * NOTICE TO USER: * - * This source code is subject to NVIDIA ownership rights under U.S. and - * international Copyright laws. Users and possessors of this source code - * are hereby granted a nonexclusive, royalty-free license to use this code + * This source code is subject to NVIDIA ownership rights under U.S. and + * international Copyright laws. Users and possessors of this source code + * are hereby granted a nonexclusive, royalty-free license to use this code * in individual and commercial software. * - * NVIDIA MAKES NO REPRESENTATION ABOUT THE SUITABILITY OF THIS SOURCE - * CODE FOR ANY PURPOSE. IT IS PROVIDED "AS IS" WITHOUT EXPRESS OR - * IMPLIED WARRANTY OF ANY KIND. NVIDIA DISCLAIMS ALL WARRANTIES WITH - * REGARD TO THIS SOURCE CODE, INCLUDING ALL IMPLIED WARRANTIES OF + * NVIDIA MAKES NO REPRESENTATION ABOUT THE SUITABILITY OF THIS SOURCE + * CODE FOR ANY PURPOSE. IT IS PROVIDED "AS IS" WITHOUT EXPRESS OR + * IMPLIED WARRANTY OF ANY KIND. NVIDIA DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOURCE CODE, INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY, NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. - * IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL, - * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS - * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE - * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE - * OR PERFORMANCE OF THIS SOURCE CODE. + * IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL, + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE + * OR PERFORMANCE OF THIS SOURCE CODE. * - * U.S. Government End Users. This source code is a "commercial item" as - * that term is defined at 48 C.F.R. 2.101 (OCT 1995), consisting of - * "commercial computer software" and "commercial computer software - * documentation" as such terms are used in 48 C.F.R. 12.212 (SEPT 1995) - * and is provided to the U.S. Government only as a commercial end item. - * Consistent with 48 C.F.R.12.212 and 48 C.F.R. 227.7202-1 through - * 227.7202-4 (JUNE 1995), all U.S. Government End Users acquire the - * source code with only those rights set forth herein. + * U.S. Government End Users. This source code is a "commercial item" as + * that term is defined at 48 C.F.R. 2.101 (OCT 1995), consisting of + * "commercial computer software" and "commercial computer software + * documentation" as such terms are used in 48 C.F.R. 12.212 (SEPT 1995) + * and is provided to the U.S. Government only as a commercial end item. + * Consistent with 48 C.F.R.12.212 and 48 C.F.R. 227.7202-1 through + * 227.7202-4 (JUNE 1995), all U.S. Government End Users acquire the + * source code with only those rights set forth herein. * - * Any use of this source code in individual and commercial software must + * Any use of this source code in individual and commercial software must * include, in the user documentation and internal comments to the code, * the above Disclaimer and U.S. Government End Users Notice. */ + #ifndef CUDA_MATH #define CUDA_MATH @@ -74,31 +74,32 @@ #undef max #undef min namespace cuda_math { -#define __attribute__(a) // to remove warnings inside math_functions.h +#define __attribute__(a) // to remove warnings inside math_functions.h #undef INT_MAX #if CUDART_VERSION < 3000 // DEVICE_BUILTIN -struct int4 { - int x, y, z, w; -}; -struct uint4 { - unsigned int x, y, z, w; -}; -struct float4 { - float x, y, z, w; -}; -struct float2 { - float x, y; -}; + struct int4 { + int x, y, z, w; + }; + struct uint4 { + unsigned int x, y, z, w; + }; + struct float4 { + float x, y, z, w; + }; + struct float2 { + float x, y; + }; + // DEVICE_BUILTIN -typedef struct int4 int4; -typedef struct uint4 uint4; -typedef struct float4 float4; -typedef struct float2 float2; + typedef struct int4 int4; + typedef struct uint4 uint4; + typedef struct float4 float4; + typedef struct float2 float2; -extern float rsqrtf(float); // CUDA 2.3 beta +extern float rsqrtf(float); // CUDA 2.3 beta #define CUDA_FLOAT_MATH_FUNCTIONS #include <device_types.h> @@ -107,36 +108,38 @@ extern float rsqrtf(float); // CUDA 2.3 beta #undef __CUDA_INTERNAL_COMPILATION__ #undef __attribute__ -// float to integer conversion -int float2int(float a, enum cudaRoundMode mode) { - return __internal_float2uint(a, mode); +// float to integer conversion +int float2int(float a, enum cudaRoundMode mode) +{ + return __internal_float2uint(a, mode); } -// float to unsigned integer conversion -unsigned int float2uint(float a, enum cudaRoundMode mode) { - return __internal_float2uint(a, mode); +// float to unsigned integer conversion +unsigned int float2uint(float a, enum cudaRoundMode mode) +{ + return __internal_float2uint(a, mode); } float __ll2float_rz(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_ru(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rd(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } #else @@ -144,211 +147,205 @@ float __ll2float_rd(long long int a) { #define CUDA_FLOAT_MATH_FUNCTIONS #define __CUDACC__ -// implementing int to float intrinsics with different rounding modes +// implementing int to float intrinsics with different rounding modes #include <device_types.h> #include <fenv.h> + // 32-bit integer to float float __int2float_rn(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_rz(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_ru(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_rd(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } // 32-bit unsigned integer to float float __uint2float_rn(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_rz(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_ru(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_rd(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } // 64-bit integer to float float __ll2float_rn(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rz(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_ru(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rd(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } -// 64-bit unsigned integer to float +// 64-bit unsigned integer to float float __ull2float_rn(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_rz(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_ru(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_rd(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } -// float to integer conversion -int float2int(float a, enum cudaRoundMode mode) { - int tmp; - switch (mode) { - case cudaRoundZero: - tmp = truncf(a); - break; - case cudaRoundNearest: - tmp = nearbyintf(a); - break; - case cudaRoundMinInf: - tmp = floorf(a); - break; - case cudaRoundPosInf: - tmp = ceilf(a); - break; - default: - abort(); - } - return tmp; +// float to integer conversion +int float2int(float a, enum cudaRoundMode mode) +{ + int tmp; + switch (mode) { + case cudaRoundZero: tmp = truncf(a); break; + case cudaRoundNearest: tmp = nearbyintf(a); break; + case cudaRoundMinInf: tmp = floorf(a); break; + case cudaRoundPosInf: tmp = ceilf(a); break; + default: abort(); + } + return tmp; } -int __internal_float2int(float a, enum cudaRoundMode mode) { - return float2int(a, mode); +int __internal_float2int(float a, enum cudaRoundMode mode) +{ + return float2int(a, mode); } -// float to unsigned integer conversion -unsigned int float2uint(float a, enum cudaRoundMode mode) { - unsigned int tmp; - switch (mode) { - case cudaRoundZero: - tmp = truncf(a); - break; - case cudaRoundNearest: - tmp = nearbyintf(a); - break; - case cudaRoundMinInf: - tmp = floorf(a); - break; - case cudaRoundPosInf: - tmp = ceilf(a); - break; - default: - abort(); - } - return tmp; +// float to unsigned integer conversion +unsigned int float2uint(float a, enum cudaRoundMode mode) +{ + unsigned int tmp; + switch (mode) { + case cudaRoundZero: tmp = truncf(a); break; + case cudaRoundNearest: tmp = nearbyintf(a); break; + case cudaRoundMinInf: tmp = floorf(a); break; + case cudaRoundPosInf: tmp = ceilf(a); break; + default: abort(); + } + return tmp; } -unsigned int __internal_float2uint(float a, enum cudaRoundMode mode) { - return float2uint(a, mode); +unsigned int __internal_float2uint(float a, enum cudaRoundMode mode) +{ + return float2uint(a, mode); } -// intrinsic for division -float fdividef(float a, float b) { return (a / b); } +// intrinsic for division +float fdividef(float a, float b) +{ + return (a / b); +} -float __internal_accurate_fdividef(float a, float b) { return fdividef(a, b); } +float __internal_accurate_fdividef(float a, float b) +{ + return fdividef(a, b); +} // intrinsic for saturate (clamp values beyond 0 and 1) -float __saturatef(float a) { - float b; - if (std::isnan(a)) - b = 0.0f; - else if (a >= 1.0f) - b = 1.0f; - else if (a <= 0.0f) - b = 0.0f; - else - b = a; - return b; +float __saturatef(float a) +{ + float b; + if (std::isnan(a)) b = 0.0f; + else if (a >= 1.0f) b = 1.0f; + else if (a <= 0.0f) b = 0.0f; + else b = a; + return b; } -// intrinsic for power -float __powf(float a, float b) { return powf(a, b); } +// intrinsic for power +float __powf(float a, float b) +{ + return powf(a, b); +} // math functions missing in Mac OSX GCC #ifdef __APPLE__ -int __signbitd(double d) { - unsigned long long int u = *((unsigned long long int*)&d); - return ((u & 0x8000000000000000ULL) != 0); +int __signbitd(double d) +{ + unsigned long long int u = *((unsigned long long int*)&d); + return ((u & 0x8000000000000000ULL) != 0); } -#endif +#endif #undef __CUDACC__ #define __CUDA_INTERNAL_COMPILATION__ @@ -357,11 +354,15 @@ int __signbitd(double d) { #undef __attribute__ #endif + } // math functions missing in Mac OSX GCC #ifdef __APPLE__ -int isnanf(float a) { return (std::isnan(a)); } -#endif +int isnanf(float a) +{ + return (std::isnan(a)); +} +#endif #endif diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index ac8af05..7a130ea 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1,5 +1,5 @@ // Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda, Wilson W.L. Fung, -// George L. Yuan, Jimmy Kwa +// George L. Yuan, Jimmy Kwa // The University of British Columbia // All rights reserved. // @@ -8,16 +8,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -33,318 +31,260 @@ #include "instructions.h" #include "ptx_ir.h" class ptx_recognizer; -typedef void *yyscan_t; +typedef void * yyscan_t; +#include "ptx.tab.h" +#include "ptx_sim.h" #include <stdio.h> -#include <map> -#include <set> #include <sstream> -#include "../../libcuda/gpgpu_context.h" -#include "../abstract_hardware_model.h" -#include "../gpgpu-sim/gpu-sim.h" -#include "../gpgpusim_entrypoint.h" +#include "opcodes.h" #include "../statwrapper.h" -#include "../stream_manager.h" -#include "cuda_device_runtime.h" -#include "decuda_pred_table/decuda_pred_table.h" +#include <set> +#include <map> +#include "../abstract_hardware_model.h" #include "memory.h" -#include "opcodes.h" #include "ptx-stats.h" -#include "ptx.tab.h" #include "ptx_loader.h" #include "ptx_parser.h" +#include "../gpgpu-sim/gpu-sim.h" #include "ptx_sim.h" -#include "ptx_sim.h" +#include "../gpgpusim_entrypoint.h" +#include "decuda_pred_table/decuda_pred_table.h" +#include "../stream_manager.h" +#include "cuda_device_runtime.h" +#include "../../libcuda/gpgpu_context.h" int g_debug_execution = 0; // Output debug information to file options -void cuda_sim::ptx_opcocde_latency_options(option_parser_t opp) { - option_parser_register(opp, "-ptx_opcode_latency_int", OPT_CSTR, - &opcode_latency_int, - "Opcode latencies for integers <ADD,MAX,MUL,MAD,DIV>" - "Default 1,1,19,25,145", - "1,1,19,25,145"); - option_parser_register(opp, "-ptx_opcode_latency_fp", OPT_CSTR, - &opcode_latency_fp, - "Opcode latencies for single precision floating " - "points <ADD,MAX,MUL,MAD,DIV>" - "Default 1,1,1,1,30", - "1,1,1,1,30"); - option_parser_register(opp, "-ptx_opcode_latency_dp", OPT_CSTR, - &opcode_latency_dp, - "Opcode latencies for double precision floating " - "points <ADD,MAX,MUL,MAD,DIV>" - "Default 8,8,8,8,335", - "8,8,8,8,335"); - option_parser_register(opp, "-ptx_opcode_latency_sfu", OPT_CSTR, - &opcode_latency_sfu, - "Opcode latencies for SFU instructions" - "Default 8", - "8"); - option_parser_register(opp, "-ptx_opcode_latency_tesnor", OPT_CSTR, - &opcode_latency_tensor, - "Opcode latencies for Tensor instructions" - "Default 64", - "64"); - option_parser_register( - opp, "-ptx_opcode_initiation_int", OPT_CSTR, &opcode_initiation_int, - "Opcode initiation intervals for integers <ADD,MAX,MUL,MAD,DIV>" - "Default 1,1,4,4,32", - "1,1,4,4,32"); - option_parser_register(opp, "-ptx_opcode_initiation_fp", OPT_CSTR, - &opcode_initiation_fp, - "Opcode initiation intervals for single precision " - "floating points <ADD,MAX,MUL,MAD,DIV>" - "Default 1,1,1,1,5", - "1,1,1,1,5"); - option_parser_register(opp, "-ptx_opcode_initiation_dp", OPT_CSTR, - &opcode_initiation_dp, - "Opcode initiation intervals for double precision " - "floating points <ADD,MAX,MUL,MAD,DIV>" - "Default 8,8,8,8,130", - "8,8,8,8,130"); - option_parser_register(opp, "-ptx_opcode_initiation_sfu", OPT_CSTR, - &opcode_initiation_sfu, - "Opcode initiation intervals for sfu instructions" - "Default 8", - "8"); - option_parser_register(opp, "-ptx_opcode_initiation_tensor", OPT_CSTR, - &opcode_initiation_tensor, - "Opcode initiation intervals for tensor instructions" - "Default 64", - "64"); - option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str, - "CDP API latency <cudaStreamCreateWithFlags, \ + +void cuda_sim::ptx_opcocde_latency_options (option_parser_t opp) { + option_parser_register(opp, "-ptx_opcode_latency_int", OPT_CSTR, &opcode_latency_int, + "Opcode latencies for integers <ADD,MAX,MUL,MAD,DIV>" + "Default 1,1,19,25,145", + "1,1,19,25,145"); + option_parser_register(opp, "-ptx_opcode_latency_fp", OPT_CSTR, &opcode_latency_fp, + "Opcode latencies for single precision floating points <ADD,MAX,MUL,MAD,DIV>" + "Default 1,1,1,1,30", + "1,1,1,1,30"); + option_parser_register(opp, "-ptx_opcode_latency_dp", OPT_CSTR, &opcode_latency_dp, + "Opcode latencies for double precision floating points <ADD,MAX,MUL,MAD,DIV>" + "Default 8,8,8,8,335", + "8,8,8,8,335"); + option_parser_register(opp, "-ptx_opcode_latency_sfu", OPT_CSTR, &opcode_latency_sfu, + "Opcode latencies for SFU instructions" + "Default 8", + "8"); + option_parser_register(opp, "-ptx_opcode_latency_tesnor", OPT_CSTR, &opcode_latency_tensor, + "Opcode latencies for Tensor instructions" + "Default 64", + "64"); + option_parser_register(opp, "-ptx_opcode_initiation_int", OPT_CSTR, &opcode_initiation_int, + "Opcode initiation intervals for integers <ADD,MAX,MUL,MAD,DIV>" + "Default 1,1,4,4,32", + "1,1,4,4,32"); + option_parser_register(opp, "-ptx_opcode_initiation_fp", OPT_CSTR, &opcode_initiation_fp, + "Opcode initiation intervals for single precision floating points <ADD,MAX,MUL,MAD,DIV>" + "Default 1,1,1,1,5", + "1,1,1,1,5"); + option_parser_register(opp, "-ptx_opcode_initiation_dp", OPT_CSTR, &opcode_initiation_dp, + "Opcode initiation intervals for double precision floating points <ADD,MAX,MUL,MAD,DIV>" + "Default 8,8,8,8,130", + "8,8,8,8,130"); + option_parser_register(opp, "-ptx_opcode_initiation_sfu", OPT_CSTR, &opcode_initiation_sfu, + "Opcode initiation intervals for sfu instructions" + "Default 8", + "8"); + option_parser_register(opp, "-ptx_opcode_initiation_tensor", OPT_CSTR, &opcode_initiation_tensor, + "Opcode initiation intervals for tensor instructions" + "Default 64", + "64"); + option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str, + "CDP API latency <cudaStreamCreateWithFlags, \ cudaGetParameterBufferV2_init_perWarp, cudaGetParameterBufferV2_perKernel, \ cudaLaunchDeviceV2_init_perWarp, cudaLaunchDevicV2_perKernel>" - "Default 7200,8000,100,12000,1600", - "7200,8000,100,12000,1600"); + "Default 7200,8000,100,12000,1600", + "7200,8000,100,12000,1600"); } -void gpgpu_t::gpgpu_ptx_sim_bindNameToTexture( - const char *name, const struct textureReference *texref, int dim, - int readmode, int ext) { - std::string texname(name); - if (m_NameToTextureRef.find(texname) == m_NameToTextureRef.end()) { - m_NameToTextureRef[texname] = std::set<const struct textureReference *>(); - } else { - const struct textureReference *tr = *m_NameToTextureRef[texname].begin(); - assert(tr != NULL); - // asserts that all texrefs in set have same fields - assert(tr->normalized == texref->normalized && - tr->filterMode == texref->filterMode && - tr->addressMode[0] == texref->addressMode[0] && - tr->addressMode[1] == texref->addressMode[1] && - tr->addressMode[2] == texref->addressMode[2] && - tr->channelDesc.x == texref->channelDesc.x && - tr->channelDesc.y == texref->channelDesc.y && - tr->channelDesc.z == texref->channelDesc.z && - tr->channelDesc.w == texref->channelDesc.w && - tr->channelDesc.f == texref->channelDesc.f); - } - m_NameToTextureRef[texname].insert(texref); - m_TextureRefToName[texref] = texname; - const textureReferenceAttr *texAttr = new textureReferenceAttr( - texref, dim, (enum cudaTextureReadMode)readmode, ext); - m_NameToAttribute[texname] = texAttr; +void gpgpu_t::gpgpu_ptx_sim_bindNameToTexture(const char* name, const struct textureReference* texref, int dim, int readmode, int ext) +{ + std::string texname(name); + if (m_NameToTextureRef.find(texname)==m_NameToTextureRef.end()){ + m_NameToTextureRef[texname] = std::set<const struct textureReference*>(); + }else{ + const struct textureReference* tr = *m_NameToTextureRef[texname].begin(); + assert(tr!=NULL); + //asserts that all texrefs in set have same fields + assert(tr->normalized==texref->normalized&& + tr->filterMode==texref->filterMode&& + tr->addressMode[0]==texref->addressMode[0]&& + tr->addressMode[1]==texref->addressMode[1]&& + tr->addressMode[2]==texref->addressMode[2]&& + tr->channelDesc.x==texref->channelDesc.x&& + tr->channelDesc.y==texref->channelDesc.y&& + tr->channelDesc.z==texref->channelDesc.z&& + tr->channelDesc.w==texref->channelDesc.w&& + tr->channelDesc.f==texref->channelDesc.f + ); + } + m_NameToTextureRef[texname].insert(texref); + m_TextureRefToName[texref] = texname; + const textureReferenceAttr *texAttr = new textureReferenceAttr(texref, dim, (enum cudaTextureReadMode)readmode, ext); + m_NameToAttribute[texname] = texAttr; } -const char *gpgpu_t::gpgpu_ptx_sim_findNamefromTexture( - const struct textureReference *texref) { - std::map<const struct textureReference *, std::string>::const_iterator t = - m_TextureRefToName.find(texref); - assert(t != m_TextureRefToName.end()); - return t->second.c_str(); +const char* gpgpu_t::gpgpu_ptx_sim_findNamefromTexture(const struct textureReference* texref) +{ + std::map<const struct textureReference*, std::string>::const_iterator t=m_TextureRefToName.find(texref); + assert( t != m_TextureRefToName.end() ); + return t->second.c_str(); } -unsigned int intLOGB2(unsigned int v) { - unsigned int shift; - unsigned int r; +unsigned int intLOGB2( unsigned int v ) { + unsigned int shift; + unsigned int r; - r = 0; + r = 0; - shift = ((v & 0xFFFF0000) != 0) << 4; - v >>= shift; - r |= shift; - shift = ((v & 0xFF00) != 0) << 3; - v >>= shift; - r |= shift; - shift = ((v & 0xF0) != 0) << 2; - v >>= shift; - r |= shift; - shift = ((v & 0xC) != 0) << 1; - v >>= shift; - r |= shift; - shift = ((v & 0x2) != 0) << 0; - v >>= shift; - r |= shift; + shift = (( v & 0xFFFF0000) != 0 ) << 4; v >>= shift; r |= shift; + shift = (( v & 0xFF00 ) != 0 ) << 3; v >>= shift; r |= shift; + shift = (( v & 0xF0 ) != 0 ) << 2; v >>= shift; r |= shift; + shift = (( v & 0xC ) != 0 ) << 1; v >>= shift; r |= shift; + shift = (( v & 0x2 ) != 0 ) << 0; v >>= shift; r |= shift; - return r; + return r; } -void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray( - const struct textureReference *texref, const struct cudaArray *array) { - std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); +void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* texref, const struct cudaArray* array) +{ + std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); - std::map<std::string, const struct cudaArray *>::const_iterator t = - m_NameToCudaArray.find(texname); - // check that there's nothing there first - if (t != m_NameToCudaArray.end()) { - printf( - "GPGPU-Sim PTX: Warning: binding to texref associated with %s, which " - "was previously bound.\nImplicitly unbinding texref associated to %s " - "first\n", - texname.c_str(), texname.c_str()); - } - m_NameToCudaArray[texname] = array; - unsigned int texel_size_bits = - array->desc.w + array->desc.x + array->desc.y + array->desc.z; - unsigned int texel_size = texel_size_bits / 8; - unsigned int Tx, Ty; - int r; + std::map<std::string,const struct cudaArray*>::const_iterator t=m_NameToCudaArray.find(texname); + //check that there's nothing there first + if(t != m_NameToCudaArray.end()){ + printf("GPGPU-Sim PTX: Warning: binding to texref associated with %s, which was previously bound.\nImplicitly unbinding texref associated to %s first\n", texname.c_str(), texname.c_str()); + } + m_NameToCudaArray[texname] = array; + unsigned int texel_size_bits = array->desc.w + array->desc.x + array->desc.y + array->desc.z; + unsigned int texel_size = texel_size_bits/8; + unsigned int Tx, Ty; + int r; - printf("GPGPU-Sim PTX: texel size = %d\n", texel_size); - printf("GPGPU-Sim PTX: texture cache linesize = %d\n", - m_function_model_config.get_texcache_linesize()); - // first determine base Tx size for given linesize - switch (m_function_model_config.get_texcache_linesize()) { - case 16: - Tx = 4; - break; - case 32: - Tx = 8; - break; - case 64: - Tx = 8; - break; - case 128: - Tx = 16; - break; - case 256: - Tx = 16; - break; - default: - printf( - "GPGPU-Sim PTX: Line size of %d bytes currently not supported.\n", - m_function_model_config.get_texcache_linesize()); + printf("GPGPU-Sim PTX: texel size = %d\n", texel_size); + printf("GPGPU-Sim PTX: texture cache linesize = %d\n", m_function_model_config.get_texcache_linesize()); + //first determine base Tx size for given linesize + switch (m_function_model_config.get_texcache_linesize()) { + case 16: Tx = 4; break; + case 32: Tx = 8; break; + case 64: Tx = 8; break; + case 128: Tx = 16; break; + case 256: Tx = 16; break; + default: + printf("GPGPU-Sim PTX: Line size of %d bytes currently not supported.\n", m_function_model_config.get_texcache_linesize()); assert(0); break; - } - r = texel_size >> 2; - // modify base Tx size to take into account size of each texel in bytes - while (r != 0) { - Tx = Tx >> 1; - r = r >> 2; - } - // by now, got the correct Tx size, calculate correct Ty size - Ty = m_function_model_config.get_texcache_linesize() / (Tx * texel_size); + } + r = texel_size >> 2; + //modify base Tx size to take into account size of each texel in bytes + while (r != 0) { + Tx = Tx >> 1; + r = r >> 2; + } + //by now, got the correct Tx size, calculate correct Ty size + Ty = m_function_model_config.get_texcache_linesize()/(Tx*texel_size); - printf( - "GPGPU-Sim PTX: Tx = %d; Ty = %d, Tx_numbits = %d, Ty_numbits = %d\n", - Tx, Ty, intLOGB2(Tx), intLOGB2(Ty)); - printf("GPGPU-Sim PTX: Texel size = %d bytes; texel_size_numbits = %d\n", - texel_size, intLOGB2(texel_size)); - printf( - "GPGPU-Sim PTX: Binding texture to array starting at devPtr32 = 0x%x\n", - array->devPtr32); - printf("GPGPU-Sim PTX: Texel size = %d bytes\n", texel_size); - struct textureInfo *texInfo = - (struct textureInfo *)malloc(sizeof(struct textureInfo)); - texInfo->Tx = Tx; - texInfo->Ty = Ty; - texInfo->Tx_numbits = intLOGB2(Tx); - texInfo->Ty_numbits = intLOGB2(Ty); - texInfo->texel_size = texel_size; - texInfo->texel_size_numbits = intLOGB2(texel_size); - m_NameToTextureInfo[texname] = texInfo; + printf("GPGPU-Sim PTX: Tx = %d; Ty = %d, Tx_numbits = %d, Ty_numbits = %d\n", Tx, Ty, intLOGB2(Tx), intLOGB2(Ty)); + printf("GPGPU-Sim PTX: Texel size = %d bytes; texel_size_numbits = %d\n", texel_size, intLOGB2(texel_size)); + printf("GPGPU-Sim PTX: Binding texture to array starting at devPtr32 = 0x%x\n", array->devPtr32); + printf("GPGPU-Sim PTX: Texel size = %d bytes\n", texel_size); + struct textureInfo* texInfo = (struct textureInfo*) malloc(sizeof(struct textureInfo)); + texInfo->Tx = Tx; + texInfo->Ty = Ty; + texInfo->Tx_numbits = intLOGB2(Tx); + texInfo->Ty_numbits = intLOGB2(Ty); + texInfo->texel_size = texel_size; + texInfo->texel_size_numbits = intLOGB2(texel_size); + m_NameToTextureInfo[texname] = texInfo; } -void gpgpu_t::gpgpu_ptx_sim_unbindTexture( - const struct textureReference *texref) { - // assumes bind-use-unbind-bind-use-unbind pattern - std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); - m_NameToCudaArray.erase(texname); - m_NameToTextureInfo.erase(texname); +void gpgpu_t::gpgpu_ptx_sim_unbindTexture(const struct textureReference* texref) +{ + //assumes bind-use-unbind-bind-use-unbind pattern + std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); + m_NameToCudaArray.erase(texname); + m_NameToTextureInfo.erase(texname); } #define MAX_INST_SIZE 8 /*bytes*/ -void function_info::ptx_assemble() { - if (m_assembled) { - return; - } +void function_info::ptx_assemble() +{ + if( m_assembled ) { + return; + } - // get the instructions into instruction memory... - unsigned num_inst = m_instructions.size(); - m_instr_mem_size = MAX_INST_SIZE * (num_inst + 1); - m_instr_mem = new ptx_instruction *[m_instr_mem_size]; + // get the instructions into instruction memory... + unsigned num_inst = m_instructions.size(); + m_instr_mem_size = MAX_INST_SIZE*(num_inst+1); + m_instr_mem = new ptx_instruction*[ m_instr_mem_size ]; - printf("GPGPU-Sim PTX: instruction assembly for function \'%s\'... ", - m_name.c_str()); - fflush(stdout); - std::list<ptx_instruction *>::iterator i; + printf("GPGPU-Sim PTX: instruction assembly for function \'%s\'... ", m_name.c_str() ); + fflush(stdout); + std::list<ptx_instruction*>::iterator i; - addr_t PC = gpgpu_ctx->func_sim->g_assemble_code_next_pc; // globally unique - // address (across - // functions) - // start function on an aligned address - for (unsigned i = 0; i < (PC % MAX_INST_SIZE); i++) - gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction *)NULL); - PC += PC % MAX_INST_SIZE; - m_start_PC = PC; + addr_t PC = gpgpu_ctx->func_sim->g_assemble_code_next_pc; // globally unique address (across functions) + // start function on an aligned address + for( unsigned i=0; i < (PC%MAX_INST_SIZE); i++ ) + gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction*)NULL); + PC += PC%MAX_INST_SIZE; + m_start_PC = PC; - addr_t n = 0; // offset in m_instr_mem - // Why s_g_pc_to_insn.size() is needed to reserve additional memory for insts? - // reserve is cumulative. - // s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + - // MAX_INST_SIZE*m_instructions.size()); - gpgpu_ctx->s_g_pc_to_insn.reserve(MAX_INST_SIZE * m_instructions.size()); - for (i = m_instructions.begin(); i != m_instructions.end(); i++) { - ptx_instruction *pI = *i; - if (pI->is_label()) { - const symbol *l = pI->get_label(); - labels[l->name()] = n; - } else { - gpgpu_ctx->func_sim->g_pc_to_finfo[PC] = this; - m_instr_mem[n] = pI; - gpgpu_ctx->s_g_pc_to_insn.push_back(pI); - assert(pI == gpgpu_ctx->s_g_pc_to_insn[PC]); - pI->set_m_instr_mem_index(n); - pI->set_PC(PC); - assert(pI->inst_size() <= MAX_INST_SIZE); - for (unsigned i = 1; i < pI->inst_size(); i++) { - gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction *)NULL); - m_instr_mem[n + i] = NULL; + addr_t n=0; // offset in m_instr_mem + //Why s_g_pc_to_insn.size() is needed to reserve additional memory for insts? reserve is cumulative. + //s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + MAX_INST_SIZE*m_instructions.size()); + gpgpu_ctx->s_g_pc_to_insn.reserve(MAX_INST_SIZE*m_instructions.size()); + for ( i=m_instructions.begin(); i != m_instructions.end(); i++ ) { + ptx_instruction *pI = *i; + if ( pI->is_label() ) { + const symbol *l = pI->get_label(); + labels[l->name()] = n; + } else { + gpgpu_ctx->func_sim->g_pc_to_finfo[PC] = this; + m_instr_mem[n] = pI; + gpgpu_ctx->s_g_pc_to_insn.push_back(pI); + assert(pI == gpgpu_ctx->s_g_pc_to_insn[PC]); + pI->set_m_instr_mem_index(n); + pI->set_PC(PC); + assert( pI->inst_size() <= MAX_INST_SIZE ); + for( unsigned i=1; i < pI->inst_size(); i++ ) { + gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction*)NULL); + m_instr_mem[n+i]=NULL; + } + n += pI->inst_size(); + PC += pI->inst_size(); } - n += pI->inst_size(); - PC += pI->inst_size(); - } - } - gpgpu_ctx->func_sim->g_assemble_code_next_pc = PC; - for (unsigned ii = 0; ii < n; - ii += m_instr_mem[ii]->inst_size()) { // handle branch instructions - ptx_instruction *pI = m_instr_mem[ii]; - if (pI->get_opcode() == BRA_OP || pI->get_opcode() == BREAKADDR_OP || - pI->get_opcode() == CALLP_OP) { - operand_info &target = pI->dst(); // get operand, e.g. target name - if (labels.find(target.name()) == labels.end()) { - printf( - "GPGPU-Sim PTX: Loader error (%s:%u): Branch label \"%s\" does not " - "appear in assembly code.", - pI->source_file(), pI->source_line(), target.name().c_str()); - abort(); + } + gpgpu_ctx->func_sim->g_assemble_code_next_pc=PC; + for ( unsigned ii=0; ii < n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions + ptx_instruction *pI = m_instr_mem[ii]; + if ( pI->get_opcode() == BRA_OP || pI->get_opcode() == BREAKADDR_OP || pI->get_opcode() == CALLP_OP) { + operand_info &target = pI->dst(); //get operand, e.g. target name + if ( labels.find(target.name()) == labels.end() ) { + printf("GPGPU-Sim PTX: Loader error (%s:%u): Branch label \"%s\" does not appear in assembly code.", + pI->source_file(),pI->source_line(), target.name().c_str() ); + abort(); + } + unsigned index = labels[ target.name() ]; //determine address from name + unsigned PC = m_instr_mem[index]->get_PC(); + m_symtab->set_label_address( target.get_symbol(), PC ); + target.set_type(label_t); } - unsigned index = labels[target.name()]; // determine address from name - unsigned PC = m_instr_mem[index]->get_PC(); - m_symtab->set_label_address(target.get_symbol(), PC); - target.set_type(label_t); - } - } - m_n = n; - printf(" done.\n"); - fflush(stdout); + } + m_n = n; + printf(" done.\n"); + fflush(stdout); -// disable pdom analysis here and do it at runtime + //disable pdom analysis here and do it at runtime #if 0 printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", m_name.c_str() ); create_basic_blocks(); @@ -383,2448 +323,2247 @@ void function_info::ptx_assemble() { #endif } -addr_t shared_to_generic(unsigned smid, addr_t addr) { - assert(addr < SHARED_MEM_SIZE_MAX); - return SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX + addr; +addr_t shared_to_generic( unsigned smid, addr_t addr ) +{ + assert( addr < SHARED_MEM_SIZE_MAX ); + return SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX + addr; +} + +addr_t global_to_generic( addr_t addr ) +{ + return addr; } -addr_t global_to_generic(addr_t addr) { return addr; } +bool isspace_shared( unsigned smid, addr_t addr ) +{ + addr_t start = SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX; + addr_t end = SHARED_GENERIC_START + (smid+1)*SHARED_MEM_SIZE_MAX; + if( (addr >= end) || (addr < start) ) + return false; + return true; +} -bool isspace_shared(unsigned smid, addr_t addr) { - addr_t start = SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX; - addr_t end = SHARED_GENERIC_START + (smid + 1) * SHARED_MEM_SIZE_MAX; - if ((addr >= end) || (addr < start)) return false; - return true; +bool isspace_global( addr_t addr ) +{ + return (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT); } -bool isspace_global(addr_t addr) { - return (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT); +memory_space_t whichspace( addr_t addr ) +{ + if( (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT) ) { + return global_space; + } else if( addr >= SHARED_GENERIC_START ) { + return shared_space; + } else { + return local_space; + } } -memory_space_t whichspace(addr_t addr) { - if ((addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT)) { - return global_space; - } else if (addr >= SHARED_GENERIC_START) { - return shared_space; - } else { - return local_space; - } +addr_t generic_to_shared( unsigned smid, addr_t addr ) +{ + assert(isspace_shared(smid,addr)); + return addr - (SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX); } -addr_t generic_to_shared(unsigned smid, addr_t addr) { - assert(isspace_shared(smid, addr)); - return addr - (SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX); +addr_t local_to_generic( unsigned smid, unsigned hwtid, addr_t addr ) +{ + assert(addr < LOCAL_MEM_SIZE_MAX); + return LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid) + addr; } -addr_t local_to_generic(unsigned smid, unsigned hwtid, addr_t addr) { - assert(addr < LOCAL_MEM_SIZE_MAX); - return LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + - (LOCAL_MEM_SIZE_MAX * hwtid) + addr; +bool isspace_local( unsigned smid, unsigned hwtid, addr_t addr ) +{ + addr_t start = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid); + addr_t end = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * (hwtid+1)); + if( (addr >= end) || (addr < start) ) + return false; + return true; } -bool isspace_local(unsigned smid, unsigned hwtid, addr_t addr) { - addr_t start = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + - (LOCAL_MEM_SIZE_MAX * hwtid); - addr_t end = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + - (LOCAL_MEM_SIZE_MAX * (hwtid + 1)); - if ((addr >= end) || (addr < start)) return false; - return true; +addr_t generic_to_local( unsigned smid, unsigned hwtid, addr_t addr ) +{ + assert(isspace_local(smid,hwtid,addr)); + return addr - (LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid)); } -addr_t generic_to_local(unsigned smid, unsigned hwtid, addr_t addr) { - assert(isspace_local(smid, hwtid, addr)); - return addr - (LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + - (LOCAL_MEM_SIZE_MAX * hwtid)); +addr_t generic_to_global( addr_t addr ) +{ + return addr; } -addr_t generic_to_global(addr_t addr) { return addr; } -void *gpgpu_t::gpu_malloc(size_t size) { - unsigned long long result = m_dev_malloc; - if (g_debug_execution >= 3) { - printf( - "GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address " - "0x%Lx\n", - size, m_dev_malloc); - fflush(stdout); - } - m_dev_malloc += size; - if (size % 256) - m_dev_malloc += (256 - size % 256); // align to 256 byte boundaries - return (void *)result; +void* gpgpu_t::gpu_malloc( size_t size ) +{ + unsigned long long result = m_dev_malloc; + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address 0x%Lx\n", size, m_dev_malloc ); + fflush(stdout); + } + m_dev_malloc += size; + if (size%256) m_dev_malloc += (256 - size%256); //align to 256 byte boundaries + return(void*) result; } -void *gpgpu_t::gpu_mallocarray(size_t size) { - unsigned long long result = m_dev_malloc; - if (g_debug_execution >= 3) { - printf( - "GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address " - "0x%Lx\n", - size, m_dev_malloc); - fflush(stdout); - } - m_dev_malloc += size; - if (size % 256) - m_dev_malloc += (256 - size % 256); // align to 256 byte boundaries - return (void *)result; +void* gpgpu_t::gpu_mallocarray( size_t size ) +{ + unsigned long long result = m_dev_malloc; + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address 0x%Lx\n", size, m_dev_malloc ); + fflush(stdout); + } + m_dev_malloc += size; + if (size%256) m_dev_malloc += (256 - size%256); //align to 256 byte boundaries + return(void*) result; } -void gpgpu_t::memcpy_to_gpu(size_t dst_start_addr, const void *src, - size_t count) { - if (g_debug_execution >= 3) { - printf( - "GPGPU-Sim PTX: copying %zu bytes from CPU[0x%Lx] to GPU[0x%Lx] ... ", - count, (unsigned long long)src, (unsigned long long)dst_start_addr); - fflush(stdout); - } - char *src_data = (char *)src; - for (unsigned n = 0; n < count; n++) - m_global_mem->write(dst_start_addr + n, 1, src_data + n, NULL, NULL); - // Copy into the performance model. - // extern gpgpu_sim* g_the_gpu; - gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(dst_start_addr, count); - if (g_debug_execution >= 3) { - printf(" done.\n"); - fflush(stdout); - } +void gpgpu_t::memcpy_to_gpu( size_t dst_start_addr, const void *src, size_t count ) +{ + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: copying %zu bytes from CPU[0x%Lx] to GPU[0x%Lx] ... ", count, (unsigned long long) src, (unsigned long long) dst_start_addr ); + fflush(stdout); + } + char *src_data = (char*)src; + for (unsigned n=0; n < count; n ++ ) + m_global_mem->write(dst_start_addr+n,1, src_data+n,NULL,NULL); + + // Copy into the performance model. + //extern gpgpu_sim* g_the_gpu; + gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(dst_start_addr, count); + if(g_debug_execution >= 3) { + printf( " done.\n"); + fflush(stdout); + } } -void gpgpu_t::memcpy_from_gpu(void *dst, size_t src_start_addr, size_t count) { - if (g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to CPU[0x%Lx] ...", - count, (unsigned long long)src_start_addr, (unsigned long long)dst); - fflush(stdout); - } - unsigned char *dst_data = (unsigned char *)dst; - for (unsigned n = 0; n < count; n++) - m_global_mem->read(src_start_addr + n, 1, dst_data + n); +void gpgpu_t::memcpy_from_gpu( void *dst, size_t src_start_addr, size_t count ) +{ + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to CPU[0x%Lx] ...", count, (unsigned long long) src_start_addr, (unsigned long long) dst ); + fflush(stdout); + } + unsigned char *dst_data = (unsigned char*)dst; + for (unsigned n=0; n < count; n ++ ) + m_global_mem->read(src_start_addr+n,1,dst_data+n); - // Copy into the performance model. - // extern gpgpu_sim* g_the_gpu; - gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(src_start_addr, count); - if (g_debug_execution >= 3) { - printf(" done.\n"); - fflush(stdout); - } + // Copy into the performance model. + //extern gpgpu_sim* g_the_gpu; + gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(src_start_addr, count); + if(g_debug_execution >= 3) { + printf( " done.\n"); + fflush(stdout); + } } -void gpgpu_t::memcpy_gpu_to_gpu(size_t dst, size_t src, size_t count) { - if (g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to GPU[0x%Lx] ...", - count, (unsigned long long)src, (unsigned long long)dst); - fflush(stdout); - } - for (unsigned n = 0; n < count; n++) { - unsigned char tmp; - m_global_mem->read(src + n, 1, &tmp); - m_global_mem->write(dst + n, 1, &tmp, NULL, NULL); - } - if (g_debug_execution >= 3) { - printf(" done.\n"); - fflush(stdout); - } +void gpgpu_t::memcpy_gpu_to_gpu( size_t dst, size_t src, size_t count ) +{ + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to GPU[0x%Lx] ...", count, + (unsigned long long) src, (unsigned long long) dst ); + fflush(stdout); + } + for (unsigned n=0; n < count; n ++ ) { + unsigned char tmp; + m_global_mem->read(src+n,1,&tmp); + m_global_mem->write(dst+n,1, &tmp,NULL,NULL); + } + if(g_debug_execution >= 3) { + printf( " done.\n"); + fflush(stdout); + } } -void gpgpu_t::gpu_memset(size_t dst_start_addr, int c, size_t count) { - if (g_debug_execution >= 3) { - printf( - "GPGPU-Sim PTX: setting %zu bytes of memory to 0x%x starting at " - "0x%Lx... ", - count, (unsigned char)c, (unsigned long long)dst_start_addr); - fflush(stdout); - } - unsigned char c_value = (unsigned char)c; - for (unsigned n = 0; n < count; n++) - m_global_mem->write(dst_start_addr + n, 1, &c_value, NULL, NULL); - if (g_debug_execution >= 3) { - printf(" done.\n"); - fflush(stdout); - } +void gpgpu_t::gpu_memset( size_t dst_start_addr, int c, size_t count ) +{ + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: setting %zu bytes of memory to 0x%x starting at 0x%Lx... ", + count, (unsigned char) c, (unsigned long long) dst_start_addr ); + fflush(stdout); + } + unsigned char c_value = (unsigned char)c; + for (unsigned n=0; n < count; n ++ ) + m_global_mem->write(dst_start_addr+n,1,&c_value,NULL,NULL); + if(g_debug_execution >= 3) { + printf( " done.\n"); + fflush(stdout); + } } -void cuda_sim::ptx_print_insn(address_type pc, FILE *fp) { - std::map<unsigned, function_info *>::iterator f = g_pc_to_finfo.find(pc); - if (f == g_pc_to_finfo.end()) { - fprintf(fp, "<no instruction at address 0x%x>", pc); - return; - } - function_info *finfo = f->second; - assert(finfo); - finfo->print_insn(pc, fp); +void cuda_sim::ptx_print_insn( address_type pc, FILE *fp ) +{ + std::map<unsigned,function_info*>::iterator f = g_pc_to_finfo.find(pc); + if( f == g_pc_to_finfo.end() ) { + fprintf(fp,"<no instruction at address 0x%x>", pc ); + return; + } + function_info *finfo = f->second; + assert( finfo ); + finfo->print_insn(pc,fp); } -std::string cuda_sim::ptx_get_insn_str(address_type pc) { - std::map<unsigned, function_info *>::iterator f = g_pc_to_finfo.find(pc); - if (f == g_pc_to_finfo.end()) { -#define STR_SIZE 255 - char buff[STR_SIZE]; - buff[STR_SIZE - 1] = '\0'; - snprintf(buff, STR_SIZE, "<no instruction at address 0x%x>", pc); - return std::string(buff); - } - function_info *finfo = f->second; - assert(finfo); - return finfo->get_insn_str(pc); +std::string cuda_sim::ptx_get_insn_str( address_type pc ) +{ + std::map<unsigned,function_info*>::iterator f = g_pc_to_finfo.find(pc); + if( f == g_pc_to_finfo.end() ) { + #define STR_SIZE 255 + char buff[STR_SIZE]; + buff[STR_SIZE - 1] = '\0'; + snprintf(buff, STR_SIZE,"<no instruction at address 0x%x>", pc ); + return std::string(buff); + } + function_info *finfo = f->second; + assert( finfo ); + return finfo->get_insn_str(pc); } -void ptx_instruction::set_fp_or_int_archop() { - oprnd_type = UN_OP; - if ((m_opcode == MEMBAR_OP) || (m_opcode == SSY_OP) || (m_opcode == BRA_OP) || - (m_opcode == BAR_OP) || (m_opcode == RET_OP) || (m_opcode == RETP_OP) || - (m_opcode == NOP_OP) || (m_opcode == EXIT_OP) || (m_opcode == CALLP_OP) || - (m_opcode == CALL_OP)) { - // do nothing - } else if ((m_opcode == CVT_OP || m_opcode == SET_OP || - m_opcode == SLCT_OP)) { - if (get_type2() == F16_TYPE || get_type2() == F32_TYPE || - get_type2() == F64_TYPE || get_type2() == FF64_TYPE) { - oprnd_type = FP_OP; - } else - oprnd_type = INT_OP; +void ptx_instruction::set_fp_or_int_archop(){ + oprnd_type=UN_OP; + if((m_opcode == MEMBAR_OP)||(m_opcode == SSY_OP )||(m_opcode == BRA_OP) || (m_opcode == BAR_OP) || (m_opcode == RET_OP) || (m_opcode == RETP_OP) || (m_opcode == NOP_OP) || (m_opcode == EXIT_OP) || (m_opcode == CALLP_OP) || (m_opcode == CALL_OP)){ + // do nothing + }else if((m_opcode == CVT_OP || m_opcode == SET_OP || m_opcode == SLCT_OP)){ + if(get_type2()==F16_TYPE || get_type2()==F32_TYPE || get_type2() == F64_TYPE || get_type2() == FF64_TYPE){ + oprnd_type= FP_OP; + }else oprnd_type=INT_OP; - } else { - if (get_type() == F16_TYPE || get_type() == F32_TYPE || - get_type() == F64_TYPE || get_type() == FF64_TYPE) { - oprnd_type = FP_OP; - } else - oprnd_type = INT_OP; - } + }else{ + if(get_type()==F16_TYPE || get_type()==F32_TYPE || get_type() == F64_TYPE || get_type() == FF64_TYPE){ + oprnd_type= FP_OP; + }else oprnd_type=INT_OP; + } } -void ptx_instruction::set_mul_div_or_other_archop() { - sp_op = OTHER_OP; - if ((m_opcode != MEMBAR_OP) && (m_opcode != SSY_OP) && (m_opcode != BRA_OP) && - (m_opcode != BAR_OP) && (m_opcode != EXIT_OP) && (m_opcode != NOP_OP) && - (m_opcode != RETP_OP) && (m_opcode != RET_OP) && (m_opcode != CALLP_OP) && - (m_opcode != CALL_OP)) { - if (get_type() == F32_TYPE || get_type() == F64_TYPE || - get_type() == FF64_TYPE) { - switch (get_opcode()) { - case MUL_OP: - case MAD_OP: - sp_op = FP_MUL_OP; - break; - case DIV_OP: - sp_op = FP_DIV_OP; - break; - case LG2_OP: - sp_op = FP_LG_OP; - break; - case RSQRT_OP: - case SQRT_OP: - sp_op = FP_SQRT_OP; - break; - case RCP_OP: - sp_op = FP_DIV_OP; - break; - case SIN_OP: - case COS_OP: - sp_op = FP_SIN_OP; - break; - case EX2_OP: - sp_op = FP_EXP_OP; - break; - default: - if ((op == ALU_OP) || (op == TENSOR_CORE_OP)) sp_op = FP__OP; - break; - } - } else { - switch (get_opcode()) { - case MUL24_OP: - case MAD24_OP: - sp_op = INT_MUL24_OP; - break; - case MUL_OP: - case MAD_OP: - if (get_type() == U32_TYPE || get_type() == S32_TYPE || - get_type() == B32_TYPE) - sp_op = INT_MUL32_OP; - else - sp_op = INT_MUL_OP; - break; - case DIV_OP: - sp_op = INT_DIV_OP; - break; - default: - if ((op == ALU_OP)) sp_op = INT__OP; - break; - } - } - } +void ptx_instruction::set_mul_div_or_other_archop(){ + sp_op=OTHER_OP; + if((m_opcode != MEMBAR_OP) && (m_opcode != SSY_OP) && (m_opcode != BRA_OP) && (m_opcode != BAR_OP) && (m_opcode != EXIT_OP) && (m_opcode != NOP_OP) && (m_opcode != RETP_OP) && (m_opcode != RET_OP) && (m_opcode != CALLP_OP) && (m_opcode != CALL_OP)){ + if(get_type()==F32_TYPE || get_type() == F64_TYPE || get_type() == FF64_TYPE){ + switch(get_opcode()){ + case MUL_OP: + case MAD_OP: + sp_op=FP_MUL_OP; + break; + case DIV_OP: + sp_op=FP_DIV_OP; + break; + case LG2_OP: + sp_op=FP_LG_OP; + break; + case RSQRT_OP: + case SQRT_OP: + sp_op=FP_SQRT_OP; + break; + case RCP_OP: + sp_op=FP_DIV_OP; + break; + case SIN_OP: + case COS_OP: + sp_op=FP_SIN_OP; + break; + case EX2_OP: + sp_op=FP_EXP_OP; + break; + default: + if((op==ALU_OP)||(op==TENSOR_CORE_OP)) + sp_op=FP__OP; + break; + + } + }else { + switch(get_opcode()){ + case MUL24_OP: + case MAD24_OP: + sp_op=INT_MUL24_OP; + break; + case MUL_OP: + case MAD_OP: + if(get_type()==U32_TYPE || get_type()==S32_TYPE || get_type()==B32_TYPE) + sp_op=INT_MUL32_OP; + else + sp_op=INT_MUL_OP; + break; + case DIV_OP: + sp_op=INT_DIV_OP; + break; + default: + if((op==ALU_OP)) + sp_op=INT__OP; + break; + } + } + } + } -void ptx_instruction::set_bar_type() { - if (m_opcode == BAR_OP) { - switch (m_barrier_op) { - case SYNC_OPTION: - bar_type = SYNC; - break; - case ARRIVE_OPTION: - bar_type = ARRIVE; - break; - case RED_OPTION: - bar_type = RED; - switch (m_atomic_spec) { - case ATOMIC_POPC: - red_type = POPC_RED; - break; - case ATOMIC_AND: - red_type = AND_RED; - break; - case ATOMIC_OR: - red_type = OR_RED; - break; - } - break; - default: - abort(); - } - } else if (m_opcode == SST_OP) { - bar_type = SYNC; - } + + +void ptx_instruction::set_bar_type() +{ + if(m_opcode==BAR_OP) { + switch(m_barrier_op){ + case SYNC_OPTION: + bar_type = SYNC; + break; + case ARRIVE_OPTION: + bar_type = ARRIVE; + break; + case RED_OPTION: + bar_type = RED; + switch(m_atomic_spec){ + case ATOMIC_POPC: + red_type = POPC_RED; + break; + case ATOMIC_AND: + red_type = AND_RED; + break; + case ATOMIC_OR: + red_type = OR_RED; + break; + } + break; + default: + abort(); + } + } + else if(m_opcode==SST_OP) { + bar_type = SYNC; + } } -void ptx_instruction::set_opcode_and_latency() { - unsigned int_latency[5]; - unsigned fp_latency[5]; - unsigned dp_latency[5]; - unsigned sfu_latency; - unsigned tensor_latency; - unsigned int_init[5]; - unsigned fp_init[5]; - unsigned dp_init[5]; - unsigned sfu_init; - unsigned tensor_init; - /* - * [0] ADD,SUB - * [1] MAX,Min - * [2] MUL - * [3] MAD - * [4] DIV - */ - sscanf(gpgpu_ctx->func_sim->opcode_latency_int, "%u,%u,%u,%u,%u", - &int_latency[0], &int_latency[1], &int_latency[2], &int_latency[3], - &int_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_fp, "%u,%u,%u,%u,%u", - &fp_latency[0], &fp_latency[1], &fp_latency[2], &fp_latency[3], - &fp_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_dp, "%u,%u,%u,%u,%u", - &dp_latency[0], &dp_latency[1], &dp_latency[2], &dp_latency[3], - &dp_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_sfu, "%u", &sfu_latency); - sscanf(gpgpu_ctx->func_sim->opcode_latency_tensor, "%u", &tensor_latency); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_int, "%u,%u,%u,%u,%u", - &int_init[0], &int_init[1], &int_init[2], &int_init[3], &int_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_fp, "%u,%u,%u,%u,%u", - &fp_init[0], &fp_init[1], &fp_init[2], &fp_init[3], &fp_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_dp, "%u,%u,%u,%u,%u", - &dp_init[0], &dp_init[1], &dp_init[2], &dp_init[3], &dp_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_sfu, "%u", &sfu_init); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_tensor, "%u", &tensor_init); - sscanf(gpgpu_ctx->func_sim->cdp_latency_str, "%u,%u,%u,%u,%u", - &gpgpu_ctx->func_sim->cdp_latency[0], - &gpgpu_ctx->func_sim->cdp_latency[1], - &gpgpu_ctx->func_sim->cdp_latency[2], - &gpgpu_ctx->func_sim->cdp_latency[3], - &gpgpu_ctx->func_sim->cdp_latency[4]); - if (!m_operands.empty()) { - std::vector<operand_info>::iterator it; - for (it = ++m_operands.begin(); it != m_operands.end(); it++) { - num_operands++; - if ((it->is_reg() || it->is_vector())) { - num_regs++; - } - } - } - op = ALU_OP; - mem_op = NOT_TEX; - initiation_interval = latency = 1; - switch (m_opcode) { - case MOV_OP: - assert(!(has_memory_read() && has_memory_write())); - if (has_memory_read()) op = LOAD_OP; - if (has_memory_write()) op = STORE_OP; - break; - case LD_OP: - op = LOAD_OP; - break; - case MMA_LD_OP: - op = TENSOR_CORE_LOAD_OP; - break; - case LDU_OP: - op = LOAD_OP; - break; - case ST_OP: - op = STORE_OP; - break; - case MMA_ST_OP: - op = TENSOR_CORE_STORE_OP; - break; - case BRA_OP: - op = BRANCH_OP; - break; - case BREAKADDR_OP: - op = BRANCH_OP; - break; - case TEX_OP: - op = LOAD_OP; - mem_op = TEX; - break; - case ATOM_OP: - op = LOAD_OP; - break; - case BAR_OP: - op = BARRIER_OP; - break; - case SST_OP: - op = BARRIER_OP; - break; - case MEMBAR_OP: - op = MEMORY_BARRIER_OP; - break; - case CALL_OP: { - if (m_is_printf || m_is_cdp) { - op = ALU_OP; - } else - op = CALL_OPS; - break; - } - case CALLP_OP: { - if (m_is_printf || m_is_cdp) { - op = ALU_OP; - } else - op = CALL_OPS; - break; - } - case RET_OP: - case RETP_OP: - op = RET_OPS; - break; - case ADD_OP: - case ADDP_OP: - case ADDC_OP: - case SUB_OP: - case SUBC_OP: - // ADD,SUB latency - switch (get_type()) { - case F32_TYPE: - latency = fp_latency[0]; - initiation_interval = fp_init[0]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[0]; - initiation_interval = dp_init[0]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: // Use int settings for default - latency = int_latency[0]; - initiation_interval = int_init[0]; - op = INTP_OP; - break; - } - break; - case MAX_OP: - case MIN_OP: - // MAX,MIN latency - switch (get_type()) { - case F32_TYPE: - latency = fp_latency[1]; - initiation_interval = fp_init[1]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[1]; - initiation_interval = dp_init[1]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: // Use int settings for default - latency = int_latency[1]; - initiation_interval = int_init[1]; - op = INTP_OP; - break; - } - break; - case MUL_OP: - // MUL latency - switch (get_type()) { - case F32_TYPE: - latency = fp_latency[2]; - initiation_interval = fp_init[2]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[2]; - initiation_interval = dp_init[2]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: // Use int settings for default - latency = int_latency[2]; - initiation_interval = int_init[2]; - op = INTP_OP; - break; - } - break; - case MAD_OP: - case MADC_OP: - case MADP_OP: - // MAD latency - switch (get_type()) { - case F32_TYPE: - latency = fp_latency[3]; - initiation_interval = fp_init[3]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[3]; - initiation_interval = dp_init[3]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: // Use int settings for default - latency = int_latency[3]; - initiation_interval = int_init[3]; - op = INTP_OP; - break; - } - break; - case DIV_OP: - // Floating point only - op = SFU_OP; - switch (get_type()) { - case F32_TYPE: - latency = fp_latency[4]; - initiation_interval = fp_init[4]; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[4]; - initiation_interval = dp_init[4]; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: // Use int settings for default - latency = int_latency[4]; - initiation_interval = int_init[4]; - break; - } - break; - case SQRT_OP: - case SIN_OP: - case COS_OP: - case EX2_OP: - case LG2_OP: - case RSQRT_OP: - case RCP_OP: - latency = sfu_latency; - initiation_interval = sfu_init; +void ptx_instruction::set_opcode_and_latency() +{ + unsigned int_latency[5]; + unsigned fp_latency[5]; + unsigned dp_latency[5]; + unsigned sfu_latency; + unsigned tensor_latency; + unsigned int_init[5]; + unsigned fp_init[5]; + unsigned dp_init[5]; + unsigned sfu_init; + unsigned tensor_init; + /* + * [0] ADD,SUB + * [1] MAX,Min + * [2] MUL + * [3] MAD + * [4] DIV + */ + sscanf(gpgpu_ctx->func_sim->opcode_latency_int, "%u,%u,%u,%u,%u", + &int_latency[0],&int_latency[1],&int_latency[2], + &int_latency[3],&int_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_fp, "%u,%u,%u,%u,%u", + &fp_latency[0],&fp_latency[1],&fp_latency[2], + &fp_latency[3],&fp_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_dp, "%u,%u,%u,%u,%u", + &dp_latency[0],&dp_latency[1],&dp_latency[2], + &dp_latency[3],&dp_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_sfu, "%u", + &sfu_latency); + sscanf(gpgpu_ctx->func_sim->opcode_latency_tensor, "%u", + &tensor_latency); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_int, "%u,%u,%u,%u,%u", + &int_init[0],&int_init[1],&int_init[2], + &int_init[3],&int_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_fp, "%u,%u,%u,%u,%u", + &fp_init[0],&fp_init[1],&fp_init[2], + &fp_init[3],&fp_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_dp, "%u,%u,%u,%u,%u", + &dp_init[0],&dp_init[1],&dp_init[2], + &dp_init[3],&dp_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_sfu, "%u", + &sfu_init); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_tensor, "%u", + &tensor_init); + sscanf(gpgpu_ctx->func_sim->cdp_latency_str, "%u,%u,%u,%u,%u", + &gpgpu_ctx->func_sim->cdp_latency[0], + &gpgpu_ctx->func_sim->cdp_latency[1], + &gpgpu_ctx->func_sim->cdp_latency[2], + &gpgpu_ctx->func_sim->cdp_latency[3], + &gpgpu_ctx->func_sim->cdp_latency[4]); + + if(!m_operands.empty()){ + std::vector<operand_info>::iterator it; + for(it=++m_operands.begin();it!=m_operands.end();it++){ + num_operands++; + if((it->is_reg() || it->is_vector())){ + num_regs++; + } + } + } + op = ALU_OP; + mem_op= NOT_TEX; + initiation_interval = latency = 1; + switch( m_opcode ) { + case MOV_OP: + assert( !(has_memory_read() && has_memory_write()) ); + if ( has_memory_read() ) op = LOAD_OP; + if ( has_memory_write() ) op = STORE_OP; + break; + case LD_OP: op = LOAD_OP; break; + case MMA_LD_OP: op = TENSOR_CORE_LOAD_OP; break; + case LDU_OP: op = LOAD_OP; break; + case ST_OP: op = STORE_OP; break; + case MMA_ST_OP: op = TENSOR_CORE_STORE_OP; break; + case BRA_OP: op = BRANCH_OP; break; + case BREAKADDR_OP: op = BRANCH_OP; break; + case TEX_OP: op = LOAD_OP; mem_op=TEX; break; + case ATOM_OP: op = LOAD_OP; break; + case BAR_OP: op = BARRIER_OP; break; + case SST_OP: op = BARRIER_OP; break; + case MEMBAR_OP: op = MEMORY_BARRIER_OP; break; + case CALL_OP: + { + if(m_is_printf || m_is_cdp) { + op = ALU_OP; + } + else + op = CALL_OPS; + break; + } + case CALLP_OP: + { + if(m_is_printf || m_is_cdp) { + op = ALU_OP; + } + else + op = CALL_OPS; + break; + } + case RET_OP: case RETP_OP: op = RET_OPS;break; + case ADD_OP: case ADDP_OP: case ADDC_OP: case SUB_OP: case SUBC_OP: + //ADD,SUB latency + switch(get_type()){ + case F32_TYPE: + latency = fp_latency[0]; + initiation_interval = fp_init[0]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[0]; + initiation_interval = dp_init[0]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: //Use int settings for default + latency = int_latency[0]; + initiation_interval = int_init[0]; + op = INTP_OP; + break; + } + break; + case MAX_OP: case MIN_OP: + //MAX,MIN latency + switch(get_type()){ + case F32_TYPE: + latency = fp_latency[1]; + initiation_interval = fp_init[1]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[1]; + initiation_interval = dp_init[1]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: //Use int settings for default + latency = int_latency[1]; + initiation_interval = int_init[1]; + op = INTP_OP; + break; + } + break; + case MUL_OP: + //MUL latency + switch(get_type()){ + case F32_TYPE: + latency = fp_latency[2]; + initiation_interval = fp_init[2]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[2]; + initiation_interval = dp_init[2]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: //Use int settings for default + latency = int_latency[2]; + initiation_interval = int_init[2]; + op = INTP_OP; + break; + } + break; + case MAD_OP: case MADC_OP: case MADP_OP: + //MAD latency + switch(get_type()){ + case F32_TYPE: + latency = fp_latency[3]; + initiation_interval = fp_init[3]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[3]; + initiation_interval = dp_init[3]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: //Use int settings for default + latency = int_latency[3]; + initiation_interval = int_init[3]; + op = INTP_OP; + break; + } + break; + case DIV_OP: + // Floating point only + op = SFU_OP; + switch(get_type()){ + case F32_TYPE: + latency = fp_latency[4]; + initiation_interval = fp_init[4]; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[4]; + initiation_interval = dp_init[4]; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: //Use int settings for default + latency = int_latency[4]; + initiation_interval = int_init[4]; + break; + } + break; + case SQRT_OP: case SIN_OP: case COS_OP: case EX2_OP: case LG2_OP: case RSQRT_OP: case RCP_OP: + latency = sfu_latency; + initiation_interval = sfu_init; op = SFU_OP; break; - case MMA_OP: - latency = tensor_latency; - initiation_interval = tensor_init; - op = TENSOR_CORE_OP; - break; - case SHFL_OP: - latency = 4; - initiation_interval = 4; - break; - default: - break; - } - set_fp_or_int_archop(); - set_mul_div_or_other_archop(); + case MMA_OP: + latency = tensor_latency; + initiation_interval = tensor_init; + op=TENSOR_CORE_OP; + break; + case SHFL_OP: + latency = 4; + initiation_interval = 4; + break; + default: + break; + } + set_fp_or_int_archop(); + set_mul_div_or_other_archop(); + } -void ptx_thread_info::ptx_fetch_inst(inst_t &inst) const { - addr_t pc = get_pc(); - const ptx_instruction *pI = m_func_info->get_instruction(pc); - inst = (const inst_t &)*pI; - assert(inst.valid()); +void ptx_thread_info::ptx_fetch_inst( inst_t &inst ) const +{ + addr_t pc = get_pc(); + const ptx_instruction *pI = m_func_info->get_instruction(pc); + inst = (const inst_t&)*pI; + assert( inst.valid() ); } -static unsigned datatype2size(unsigned data_type) { - unsigned data_size; - switch (data_type) { - case B8_TYPE: - case S8_TYPE: - case U8_TYPE: - data_size = 1; - break; - case B16_TYPE: - case S16_TYPE: - case U16_TYPE: - case F16_TYPE: - data_size = 2; - break; - case B32_TYPE: - case S32_TYPE: - case U32_TYPE: - case F32_TYPE: - data_size = 4; - break; - case B64_TYPE: - case BB64_TYPE: - case S64_TYPE: - case U64_TYPE: - case F64_TYPE: - case FF64_TYPE: - data_size = 8; - break; - case BB128_TYPE: - data_size = 16; - break; - default: - assert(0); - break; - } - return data_size; +static unsigned datatype2size( unsigned data_type ) +{ + unsigned data_size; + switch ( data_type ) { + case B8_TYPE: + case S8_TYPE: + case U8_TYPE: + data_size = 1; break; + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: + case F16_TYPE: + data_size = 2; break; + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: + case F32_TYPE: + data_size = 4; break; + case B64_TYPE: + case BB64_TYPE: + case S64_TYPE: + case U64_TYPE: + case F64_TYPE: + case FF64_TYPE: + data_size = 8; break; + case BB128_TYPE: + data_size = 16; break; + default: assert(0); break; + } + return data_size; } -void ptx_instruction::pre_decode() { - pc = m_PC; - isize = m_inst_size; - for (unsigned i = 0; i < MAX_OUTPUT_VALUES; i++) { - out[i] = 0; - } - for (unsigned i = 0; i < MAX_INPUT_VALUES; i++) { - in[i] = 0; - } - incount = 0; - outcount = 0; - is_vectorin = 0; - is_vectorout = 0; - std::fill_n(arch_reg.src, MAX_REG_OPERANDS, -1); - std::fill_n(arch_reg.dst, MAX_REG_OPERANDS, -1); - pred = 0; - ar1 = 0; - ar2 = 0; - space = m_space_spec; - memory_op = no_memory_op; - data_size = 0; - if (has_memory_read() || has_memory_write()) { - unsigned to_type = get_type(); - data_size = datatype2size(to_type); - memory_op = has_memory_read() ? memory_load : memory_store; - } +void ptx_instruction::pre_decode() +{ + pc = m_PC; + isize = m_inst_size; + for(unsigned i=0; i<MAX_OUTPUT_VALUES; i++) { + out[i] = 0; + } + for(unsigned i=0; i<MAX_INPUT_VALUES; i++) { + in[i] = 0; + } + incount=0; + outcount=0; + is_vectorin = 0; + is_vectorout = 0; + std::fill_n(arch_reg.src, MAX_REG_OPERANDS, -1); + std::fill_n(arch_reg.dst, MAX_REG_OPERANDS, -1); + pred = 0; + ar1 = 0; + ar2 = 0; + space = m_space_spec; + memory_op = no_memory_op; + data_size = 0; + if ( has_memory_read() || has_memory_write() ) { + unsigned to_type = get_type(); + data_size = datatype2size(to_type); + memory_op = has_memory_read() ? memory_load : memory_store; + } - bool has_dst = false; + bool has_dst = false ; - switch (get_opcode()) { -#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ - case OP: \ - has_dst = (DST != 0); \ - break; -#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ - case OP: \ - has_dst = (DST != 0); \ - break; + switch ( get_opcode() ) { +#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break; +#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break; #include "opcodes.def" #undef OP_DEF #undef OP_W_DEF - default: - printf("Execution error: Invalid opcode (0x%x)\n", get_opcode()); + default: + printf( "Execution error: Invalid opcode (0x%x)\n", get_opcode() ); break; - } + } - switch (m_cache_option) { - case CA_OPTION: - cache_op = CACHE_ALL; - break; - case NC_OPTION: - cache_op = CACHE_L1; - break; - case CG_OPTION: - cache_op = CACHE_GLOBAL; - break; - case CS_OPTION: - cache_op = CACHE_STREAMING; + switch( m_cache_option ) { + case CA_OPTION: cache_op = CACHE_ALL; break; + case NC_OPTION: cache_op = CACHE_L1; break; + case CG_OPTION: cache_op = CACHE_GLOBAL; break; + case CS_OPTION: cache_op = CACHE_STREAMING; break; + case LU_OPTION: cache_op = CACHE_LAST_USE; break; + case CV_OPTION: cache_op = CACHE_VOLATILE; break; + case WB_OPTION: cache_op = CACHE_WRITE_BACK; break; + case WT_OPTION: cache_op = CACHE_WRITE_THROUGH; break; + default: + //if( m_opcode == LD_OP || m_opcode == LDU_OP ) + if( m_opcode == MMA_LD_OP || m_opcode == LD_OP || m_opcode == LDU_OP ) + cache_op = CACHE_ALL; + //else if( m_opcode == ST_OP ) + else if( m_opcode == MMA_ST_OP || m_opcode == ST_OP ) + cache_op = CACHE_WRITE_BACK; + else if( m_opcode == ATOM_OP ) + cache_op = CACHE_GLOBAL; break; - case LU_OPTION: - cache_op = CACHE_LAST_USE; - break; - case CV_OPTION: - cache_op = CACHE_VOLATILE; - break; - case WB_OPTION: - cache_op = CACHE_WRITE_BACK; - break; - case WT_OPTION: - cache_op = CACHE_WRITE_THROUGH; - break; - default: - // if( m_opcode == LD_OP || m_opcode == LDU_OP ) - if (m_opcode == MMA_LD_OP || m_opcode == LD_OP || m_opcode == LDU_OP) - cache_op = CACHE_ALL; - // else if( m_opcode == ST_OP ) - else if (m_opcode == MMA_ST_OP || m_opcode == ST_OP) - cache_op = CACHE_WRITE_BACK; - else if (m_opcode == ATOM_OP) - cache_op = CACHE_GLOBAL; - break; - } + } - set_opcode_and_latency(); - set_bar_type(); - // Get register operands - int n = 0, m = 0; - ptx_instruction::const_iterator opr = op_iter_begin(); - for (; opr != op_iter_end(); opr++, n++) { // process operands - const operand_info &o = *opr; - if (has_dst && n == 0) { - // Do not set the null register "_" as an architectural register - if (o.is_reg() && !o.is_non_arch_reg()) { - out[0] = o.reg_num(); - arch_reg.dst[0] = o.arch_reg_num(); - } else if (o.is_vector()) { - is_vectorin = 1; - unsigned num_elem = o.get_vect_nelem(); - if (num_elem >= 1) out[0] = o.reg1_num(); - if (num_elem >= 2) out[1] = o.reg2_num(); - if (num_elem >= 3) out[2] = o.reg3_num(); - if (num_elem >= 4) out[3] = o.reg4_num(); - if (num_elem >= 5) out[4] = o.reg5_num(); - if (num_elem >= 6) out[5] = o.reg6_num(); - if (num_elem >= 7) out[6] = o.reg7_num(); - if (num_elem >= 8) out[7] = o.reg8_num(); - for (int i = 0; i < num_elem; i++) arch_reg.dst[i] = o.arch_reg_num(i); - } - } else { - if (o.is_reg() && !o.is_non_arch_reg()) { - int reg_num = o.reg_num(); - arch_reg.src[m] = o.arch_reg_num(); - switch (m) { - case 0: - in[0] = reg_num; - break; - case 1: - in[1] = reg_num; - break; - case 2: - in[2] = reg_num; - break; - default: - break; - } - m++; - } else if (o.is_vector()) { - // assert(m == 0); //only support 1 vector operand (for textures) right - // now - is_vectorout = 1; - unsigned num_elem = o.get_vect_nelem(); - if (num_elem >= 1) in[m + 0] = o.reg1_num(); - if (num_elem >= 2) in[m + 1] = o.reg2_num(); - if (num_elem >= 3) in[m + 2] = o.reg3_num(); - if (num_elem >= 4) in[m + 3] = o.reg4_num(); - if (num_elem >= 5) in[m + 4] = o.reg5_num(); - if (num_elem >= 6) in[m + 5] = o.reg6_num(); - if (num_elem >= 7) in[m + 6] = o.reg7_num(); - if (num_elem >= 8) in[m + 7] = o.reg8_num(); - for (int i = 0; i < num_elem; i++) - arch_reg.src[m + i] = o.arch_reg_num(i); - m += num_elem; + set_opcode_and_latency(); + set_bar_type(); + // Get register operands + int n=0,m=0; + ptx_instruction::const_iterator opr=op_iter_begin(); + for ( ; opr != op_iter_end(); opr++, n++ ) { //process operands + const operand_info &o = *opr; + if ( has_dst && n==0 ) { + // Do not set the null register "_" as an architectural register + if ( o.is_reg() && !o.is_non_arch_reg() ) { + out[0] = o.reg_num(); + arch_reg.dst[0] = o.arch_reg_num(); + } else if ( o.is_vector() ) { + is_vectorin = 1; + unsigned num_elem = o.get_vect_nelem(); + if( num_elem >= 1 ) out[0] = o.reg1_num(); + if( num_elem >= 2 ) out[1] = o.reg2_num(); + if( num_elem >= 3 ) out[2] = o.reg3_num(); + if( num_elem >= 4 ) out[3] = o.reg4_num(); + if( num_elem >= 5 ) out[4] = o.reg5_num(); + if( num_elem >= 6 ) out[5] = o.reg6_num(); + if( num_elem >= 7 ) out[6] = o.reg7_num(); + if( num_elem >= 8 ) out[7] = o.reg8_num(); + for (int i = 0; i < num_elem; i++) + arch_reg.dst[i] = o.arch_reg_num(i); + } + } else { + if ( o.is_reg() && !o.is_non_arch_reg() ) { + int reg_num = o.reg_num(); + arch_reg.src[m] = o.arch_reg_num(); + switch ( m ) { + case 0: in[0] = reg_num; break; + case 1: in[1] = reg_num; break; + case 2: in[2] = reg_num; break; + default: break; + } + m++; + } else if ( o.is_vector() ) { + //assert(m == 0); //only support 1 vector operand (for textures) right now + is_vectorout = 1; + unsigned num_elem = o.get_vect_nelem(); + if( num_elem >= 1 ) in[m+0] = o.reg1_num(); + if( num_elem >= 2 ) in[m+1] = o.reg2_num(); + if( num_elem >= 3 ) in[m+2] = o.reg3_num(); + if( num_elem >= 4 ) in[m+3] = o.reg4_num(); + if( num_elem >= 5 ) in[m+4] = o.reg5_num(); + if( num_elem >= 6 ) in[m+5] = o.reg6_num(); + if( num_elem >= 7 ) in[m+6] = o.reg7_num(); + if( num_elem >= 8 ) in[m+7] = o.reg8_num(); + for (int i = 0; i < num_elem; i++) + arch_reg.src[m+i] = o.arch_reg_num(i); + m+=num_elem; + } } - } - } + } + + //Setting number of input and output operands which is required for scoreboard check + for(int i=0;i<MAX_OUTPUT_VALUES;i++) + if(out[i]>0) + outcount++; + + for(int i=0;i<MAX_INPUT_VALUES;i++) + if(in[i]>0) + incount++; - // Setting number of input and output operands which is required for - // scoreboard check - for (int i = 0; i < MAX_OUTPUT_VALUES; i++) - if (out[i] > 0) outcount++; + // Get predicate + if(has_pred()) { + const operand_info &p = get_pred(); + pred = p.reg_num(); + } - for (int i = 0; i < MAX_INPUT_VALUES; i++) - if (in[i] > 0) incount++; + // Get address registers inside memory operands. + // Assuming only one memory operand per instruction, + // and maximum of two address registers for one memory operand. + if( has_memory_read() || has_memory_write() ) { + ptx_instruction::const_iterator op=op_iter_begin(); + for ( ; op != op_iter_end(); op++, n++ ) { //process operands + const operand_info &o = *op; - // Get predicate - if (has_pred()) { - const operand_info &p = get_pred(); - pred = p.reg_num(); - } + if(o.is_memory_operand()) { + // We do not support the null register as a memory operand + assert( !o.is_non_arch_reg() ); - // Get address registers inside memory operands. - // Assuming only one memory operand per instruction, - // and maximum of two address registers for one memory operand. - if (has_memory_read() || has_memory_write()) { - ptx_instruction::const_iterator op = op_iter_begin(); - for (; op != op_iter_end(); op++, n++) { // process operands - const operand_info &o = *op; + // Check PTXPlus-type operand + // memory operand with addressing (ex. s[0x4] or g[$r1]) + if(o.is_memory_operand2()) { - if (o.is_memory_operand()) { - // We do not support the null register as a memory operand - assert(!o.is_non_arch_reg()); + // memory operand with one address register (ex. g[$r1+0x4] or s[$r2+=0x4]) + if(o.get_double_operand_type() == 0 || o.get_double_operand_type() == 3){ + ar1 = o.reg_num(); + arch_reg.src[4] = o.arch_reg_num(); + // TODO: address register in $r2+=0x4 should be an output register as well + } + // memory operand with two address register (ex. s[$r1+$r1] or g[$r1+=$r2]) + else if(o.get_double_operand_type() == 1 || o.get_double_operand_type() == 2) { + ar1 = o.reg1_num(); + arch_reg.src[4] = o.arch_reg_num(); + ar2 = o.reg2_num(); + arch_reg.src[5] = o.arch_reg_num(); + // TODO: first address register in $r1+=$r2 should be an output register as well + } + } + else if(o.is_immediate_address()){ - // Check PTXPlus-type operand - // memory operand with addressing (ex. s[0x4] or g[$r1]) - if (o.is_memory_operand2()) { - // memory operand with one address register (ex. g[$r1+0x4] or - // s[$r2+=0x4]) - if (o.get_double_operand_type() == 0 || - o.get_double_operand_type() == 3) { - ar1 = o.reg_num(); - arch_reg.src[4] = o.arch_reg_num(); - // TODO: address register in $r2+=0x4 should be an output register - // as well - } - // memory operand with two address register (ex. s[$r1+$r1] or - // g[$r1+=$r2]) - else if (o.get_double_operand_type() == 1 || - o.get_double_operand_type() == 2) { - ar1 = o.reg1_num(); - arch_reg.src[4] = o.arch_reg_num(); - ar2 = o.reg2_num(); - arch_reg.src[5] = o.arch_reg_num(); - // TODO: first address register in $r1+=$r2 should be an output - // register as well - } - } else if (o.is_immediate_address()) { - } - // Regular PTX operand - else if (o.get_symbol() - ->type() - ->get_key() - .is_reg()) { // Memory operand contains a register - ar1 = o.reg_num(); - arch_reg.src[4] = o.arch_reg_num(); - } + } + // Regular PTX operand + else if (o.get_symbol()->type()->get_key().is_reg()) { // Memory operand contains a register + ar1 = o.reg_num(); + arch_reg.src[4] = o.arch_reg_num(); + } + + } } - } - } + } - // get reconvergence pc - reconvergence_pc = gpgpu_ctx->func_sim->get_converge_point(pc); + // get reconvergence pc + reconvergence_pc = gpgpu_ctx->func_sim->get_converge_point(pc); - m_decoded = true; + m_decoded=true; } -void function_info::add_param_name_type_size(unsigned index, std::string name, - int type, size_t size, bool ptr, - memory_space_t space) { - unsigned parsed_index; - char buffer[2048]; - snprintf(buffer, 2048, "%s_param_%%u", m_name.c_str()); - int ntokens = sscanf(name.c_str(), buffer, &parsed_index); - if (ntokens == 1) { - assert(m_ptx_kernel_param_info.find(parsed_index) == - m_ptx_kernel_param_info.end()); - m_ptx_kernel_param_info[parsed_index] = - param_info(name, type, size, ptr, space); - } else { - assert(m_ptx_kernel_param_info.find(index) == - m_ptx_kernel_param_info.end()); - m_ptx_kernel_param_info[index] = param_info(name, type, size, ptr, space); - } +void function_info::add_param_name_type_size( unsigned index, std::string name, int type, size_t size, bool ptr, memory_space_t space ) +{ + unsigned parsed_index; + char buffer[2048]; + snprintf(buffer,2048,"%s_param_%%u", m_name.c_str() ); + int ntokens = sscanf(name.c_str(),buffer,&parsed_index); + if( ntokens == 1 ) { + assert( m_ptx_kernel_param_info.find(parsed_index) == m_ptx_kernel_param_info.end() ); + m_ptx_kernel_param_info[parsed_index] = param_info(name, type, size, ptr, space); + } else { + assert( m_ptx_kernel_param_info.find(index) == m_ptx_kernel_param_info.end() ); + m_ptx_kernel_param_info[index] = param_info(name, type, size, ptr, space); + } } -void function_info::add_param_data(unsigned argn, - struct gpgpu_ptx_sim_arg *args) { - const void *data = args->m_start; - - bool scratchpad_memory_param = - false; // Is this parameter in CUDA shared memory or OpenCL local memory - - std::map<unsigned, param_info>::iterator i = - m_ptx_kernel_param_info.find(argn); - if (i != m_ptx_kernel_param_info.end()) { - if (i->second.is_ptr_shared()) { - assert( - args->m_start == NULL && - "OpenCL parameter pointer to local memory must have NULL as value"); - scratchpad_memory_param = true; - } else { - param_t tmp; - tmp.pdata = args->m_start; - tmp.size = args->m_nbytes; - tmp.offset = args->m_offset; - tmp.type = 0; - i->second.add_data(tmp); - i->second.add_offset((unsigned)args->m_offset); - } - } else { - scratchpad_memory_param = true; - } +void function_info::add_param_data( unsigned argn, struct gpgpu_ptx_sim_arg *args ) +{ + const void *data = args->m_start; - if (scratchpad_memory_param) { - // This should only happen for OpenCL: - // - // The LLVM PTX compiler in NVIDIA's driver (version 190.29) - // does not generate an argument in the function declaration - // for __constant arguments. - // - // The associated constant memory space can be allocated in two - // ways. It can be explicitly initialized in the .ptx file where - // it is declared. Or, it can be allocated using the clCreateBuffer - // on the host. In this later case, the .ptx file will contain - // a global declaration of the parameter, but it will have an unknown - // array size. Thus, the symbol's address will not be set and we need - // to set it here before executing the PTX. + bool scratchpad_memory_param = false; // Is this parameter in CUDA shared memory or OpenCL local memory - char buffer[2048]; - snprintf(buffer, 2048, "%s_param_%u", m_name.c_str(), argn); + std::map<unsigned,param_info>::iterator i=m_ptx_kernel_param_info.find(argn); + if( i != m_ptx_kernel_param_info.end() ) { + if (i->second.is_ptr_shared()) { + assert(args->m_start == NULL && "OpenCL parameter pointer to local memory must have NULL as value"); + scratchpad_memory_param = true; + } else { + param_t tmp; + tmp.pdata = args->m_start; + tmp.size = args->m_nbytes; + tmp.offset = args->m_offset; + tmp.type = 0; + i->second.add_data(tmp); + i->second.add_offset((unsigned) args->m_offset); + } + } else { + scratchpad_memory_param = true; + } - symbol *p = m_symtab->lookup(buffer); - if (p == NULL) { - printf( - "GPGPU-Sim PTX: ERROR ** could not locate symbol for \'%s\' : cannot " - "bind buffer\n", - buffer); - abort(); - } - if (data) - p->set_address((addr_t) * (size_t *)data); - else { - // clSetKernelArg was passed NULL pointer for data... - // this is used for dynamically sized shared memory on NVIDIA platforms - bool is_ptr_shared = false; - if (i != m_ptx_kernel_param_info.end()) { - is_ptr_shared = i->second.is_ptr_shared(); + if (scratchpad_memory_param) { + // This should only happen for OpenCL: + // + // The LLVM PTX compiler in NVIDIA's driver (version 190.29) + // does not generate an argument in the function declaration + // for __constant arguments. + // + // The associated constant memory space can be allocated in two + // ways. It can be explicitly initialized in the .ptx file where + // it is declared. Or, it can be allocated using the clCreateBuffer + // on the host. In this later case, the .ptx file will contain + // a global declaration of the parameter, but it will have an unknown + // array size. Thus, the symbol's address will not be set and we need + // to set it here before executing the PTX. + + char buffer[2048]; + snprintf(buffer,2048,"%s_param_%u",m_name.c_str(),argn); + + symbol *p = m_symtab->lookup(buffer); + if( p == NULL ) { + printf("GPGPU-Sim PTX: ERROR ** could not locate symbol for \'%s\' : cannot bind buffer\n", buffer); + abort(); } + if( data ) + p->set_address((addr_t)*(size_t*)data); + else { + // clSetKernelArg was passed NULL pointer for data... + // this is used for dynamically sized shared memory on NVIDIA platforms + bool is_ptr_shared = false; + if( i != m_ptx_kernel_param_info.end() ) { + is_ptr_shared = i->second.is_ptr_shared(); + } - if (!is_ptr_shared and !p->is_shared()) { - printf( - "GPGPU-Sim PTX: ERROR ** clSetKernelArg passed NULL but arg not " - "shared memory\n"); - abort(); + if( !is_ptr_shared and !p->is_shared() ) { + printf("GPGPU-Sim PTX: ERROR ** clSetKernelArg passed NULL but arg not shared memory\n"); + abort(); + } + unsigned num_bits = 8*args->m_nbytes; + printf("GPGPU-Sim PTX: deferred allocation of shared region for \"%s\" from 0x%x to 0x%x (shared memory space)\n", + p->name().c_str(), + m_symtab->get_shared_next(), + m_symtab->get_shared_next() + num_bits/8 ); + fflush(stdout); + assert( (num_bits%8) == 0 ); + addr_t addr = m_symtab->get_shared_next(); + addr_t addr_pad = num_bits ? (((num_bits/8) - (addr % (num_bits/8))) % (num_bits/8)) : 0; + p->set_address( addr+addr_pad ); + m_symtab->alloc_shared( num_bits/8 + addr_pad ); } - unsigned num_bits = 8 * args->m_nbytes; - printf( - "GPGPU-Sim PTX: deferred allocation of shared region for \"%s\" from " - "0x%x to 0x%x (shared memory space)\n", - p->name().c_str(), m_symtab->get_shared_next(), - m_symtab->get_shared_next() + num_bits / 8); - fflush(stdout); - assert((num_bits % 8) == 0); - addr_t addr = m_symtab->get_shared_next(); - addr_t addr_pad = - num_bits - ? (((num_bits / 8) - (addr % (num_bits / 8))) % (num_bits / 8)) - : 0; - p->set_address(addr + addr_pad); - m_symtab->alloc_shared(num_bits / 8 + addr_pad); - } - } + } } unsigned function_info::get_args_aligned_size() { - if (m_args_aligned_size >= 0) return m_args_aligned_size; + + if(m_args_aligned_size >= 0) + return m_args_aligned_size; + + unsigned param_address = 0; + unsigned int total_size = 0; + for( std::map<unsigned,param_info>::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); - unsigned param_address = 0; - unsigned int total_size = 0; - for (std::map<unsigned, param_info>::iterator i = - m_ptx_kernel_param_info.begin(); - i != m_ptx_kernel_param_info.end(); i++) { - param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); + size_t arg_size = p.get_size() / 8; // size of param in bytes + total_size = (total_size + arg_size - 1) / arg_size * arg_size; //aligned + p.add_offset(total_size); + param->set_address(param_address + total_size); + total_size += arg_size; + } - size_t arg_size = p.get_size() / 8; // size of param in bytes - total_size = (total_size + arg_size - 1) / arg_size * arg_size; // aligned - p.add_offset(total_size); - param->set_address(param_address + total_size); - total_size += arg_size; - } + m_args_aligned_size = (total_size + 3) / 4 * 4; //final size aligned to word - m_args_aligned_size = (total_size + 3) / 4 * 4; // final size aligned to word + return m_args_aligned_size; - return m_args_aligned_size; } -void function_info::finalize(memory_space *param_mem) { - unsigned param_address = 0; - for (std::map<unsigned, param_info>::iterator i = - m_ptx_kernel_param_info.begin(); - i != m_ptx_kernel_param_info.end(); i++) { - param_info &p = i->second; - if (p.is_ptr_shared()) - continue; // Pointer to local memory: Should we pass the allocated shared - // memory address to the param memory space? - std::string name = p.get_name(); - int type = p.get_type(); - param_t param_value = p.get_value(); - param_value.type = type; - symbol *param = m_symtab->lookup(name.c_str()); - unsigned xtype = param->type()->get_key().scalar_type(); - assert(xtype == (unsigned)type); - size_t size; - size = param_value.size; // size of param in bytes - // assert(param_value.offset == param_address); - if (size != p.get_size() / 8) { - printf( - "GPGPU-Sim PTX: WARNING actual kernel paramter size = %zu bytes vs. " - "formal size = %zu (using smaller of two)\n", - size, p.get_size() / 8); - size = (size < (p.get_size() / 8)) ? size : (p.get_size() / 8); - } - // copy the parameter over word-by-word so that parameter that crosses a - // memory page can be copied over - // Jin: copy parameter using aligned rules - const type_info *paramtype = param->type(); - int align_amount = paramtype->get_key().get_alignment_spec(); - align_amount = (align_amount == -1) ? size : align_amount; - param_address = (param_address + align_amount - 1) / align_amount * - align_amount; // aligned - const size_t word_size = 4; - // param_address = (param_address + size - 1) / size * size; //aligned with - // size - for (size_t idx = 0; idx < size; idx += word_size) { - const char *pdata = reinterpret_cast<const char *>(param_value.pdata) + - idx; // cast to char * for ptr arithmetic - param_mem->write(param_address + idx, word_size, pdata, NULL, NULL); - } - unsigned offset = p.get_offset(); - assert(offset == param_address); - param->set_address(param_address); - param_address += size; - } +void function_info::finalize( memory_space *param_mem ) +{ + unsigned param_address = 0; + for( std::map<unsigned,param_info>::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + param_info &p = i->second; + if (p.is_ptr_shared()) continue; // Pointer to local memory: Should we pass the allocated shared memory address to the param memory space? + std::string name = p.get_name(); + int type = p.get_type(); + param_t param_value = p.get_value(); + param_value.type = type; + symbol *param = m_symtab->lookup(name.c_str()); + unsigned xtype = param->type()->get_key().scalar_type(); + assert(xtype==(unsigned)type); + size_t size; + size = param_value.size; // size of param in bytes + // assert(param_value.offset == param_address); + if( size != p.get_size() / 8) { + printf("GPGPU-Sim PTX: WARNING actual kernel paramter size = %zu bytes vs. formal size = %zu (using smaller of two)\n", + size, p.get_size()/8); + size = (size<(p.get_size()/8))?size:(p.get_size()/8); + } + // copy the parameter over word-by-word so that parameter that crosses a memory page can be copied over + //Jin: copy parameter using aligned rules + const type_info *paramtype = param->type(); + int align_amount = paramtype->get_key().get_alignment_spec(); + align_amount = (align_amount == -1) ? size : align_amount; + param_address = (param_address + align_amount - 1) / align_amount * align_amount; //aligned + + const size_t word_size = 4; + //param_address = (param_address + size - 1) / size * size; //aligned with size + for (size_t idx = 0; idx < size; idx += word_size) { + const char *pdata = reinterpret_cast<const char*>(param_value.pdata) + idx; // cast to char * for ptr arithmetic + param_mem->write(param_address + idx, word_size, pdata,NULL,NULL); + } + unsigned offset = p.get_offset(); + assert(offset == param_address); + param->set_address(param_address); + param_address += size; + } } -void function_info::param_to_shared(memory_space *shared_mem, - symbol_table *symtab) { - // TODO: call this only for PTXPlus with GT200 models - // extern gpgpu_sim* g_the_gpu; - if (not gpgpu_ctx->the_gpgpusim->g_the_gpu->get_config().convert_to_ptxplus()) - return; +void function_info::param_to_shared( memory_space *shared_mem, symbol_table *symtab ) +{ + // TODO: call this only for PTXPlus with GT200 models + //extern gpgpu_sim* g_the_gpu; + if (not gpgpu_ctx->the_gpgpusim->g_the_gpu->get_config().convert_to_ptxplus()) return; - // copies parameters into simulated shared memory - for (std::map<unsigned, param_info>::iterator i = - m_ptx_kernel_param_info.begin(); - i != m_ptx_kernel_param_info.end(); i++) { - param_info &p = i->second; - if (p.is_ptr_shared()) - continue; // Pointer to local memory: Should we pass the allocated shared - // memory address to the param memory space? - std::string name = p.get_name(); - int type = p.get_type(); - param_t value = p.get_value(); - value.type = type; - symbol *param = symtab->lookup(name.c_str()); - unsigned xtype = param->type()->get_key().scalar_type(); - assert(xtype == (unsigned)type); + // copies parameters into simulated shared memory + for( std::map<unsigned,param_info>::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + param_info &p = i->second; + if (p.is_ptr_shared()) continue; // Pointer to local memory: Should we pass the allocated shared memory address to the param memory space? + std::string name = p.get_name(); + int type = p.get_type(); + param_t value = p.get_value(); + value.type = type; + symbol *param = symtab->lookup(name.c_str()); + unsigned xtype = param->type()->get_key().scalar_type(); + assert(xtype==(unsigned)type); - int tmp; - size_t size; - unsigned offset = p.get_offset(); - type_info_key::type_decode(xtype, size, tmp); + int tmp; + size_t size; + unsigned offset = p.get_offset(); + type_info_key::type_decode(xtype,size,tmp); - // Write to shared memory - offset + 0x10 - shared_mem->write(offset + 0x10, size / 8, value.pdata, NULL, NULL); - } + // Write to shared memory - offset + 0x10 + shared_mem->write(offset+0x10,size/8,value.pdata,NULL,NULL); + } } -void function_info::list_param(FILE *fout) const { - for (std::map<unsigned, param_info>::const_iterator i = - m_ptx_kernel_param_info.begin(); - i != m_ptx_kernel_param_info.end(); i++) { - const param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); - addr_t param_addr = param->get_address(); - fprintf(fout, "%s: %#08x\n", name.c_str(), param_addr); - } - fflush(fout); + +void function_info::list_param( FILE *fout ) const +{ + for( std::map<unsigned,param_info>::const_iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + const param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); + fprintf(fout, "%s: %#08x\n", name.c_str(), param_addr); + } + fflush(fout); } -void function_info::ptx_jit_config( - std::map<unsigned long long, size_t> mallocPtr_Size, - memory_space *param_mem, gpgpu_t *gpu, dim3 gridDim, dim3 blockDim) { - static unsigned long long counter = 0; - std::vector<std::pair<size_t, unsigned char *> > param_data; - std::vector<unsigned> offsets; - std::vector<bool> paramIsPointer; +void function_info::ptx_jit_config(std::map<unsigned long long, size_t> mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) +{ + static unsigned long long counter = 0; + std::vector< std::pair<size_t, unsigned char*> > param_data; + std::vector<unsigned> offsets; + std::vector<bool> paramIsPointer; - char *gpgpusim_path = getenv("GPGPUSIM_ROOT"); - assert(gpgpusim_path != NULL); - char *wys_exec_path = getenv("WYS_EXEC_PATH"); - assert(wys_exec_path != NULL); - std::string command = - std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; - std::string filename(std::string(gpgpusim_path) + - "/debug_tools/WatchYourStep/data/params.config" + - std::to_string(counter)); + char * gpgpusim_path = getenv("GPGPUSIM_ROOT"); + assert(gpgpusim_path!=NULL); + char * wys_exec_path = getenv("WYS_EXEC_PATH"); + assert(wys_exec_path!=NULL); + std::string command = std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; + std::string filename(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/params.config" + std::to_string(counter)); - // initialize paramList - char buff[1024]; - std::string filename_c(filename + "_c"); - snprintf(buff, 1024, "c++filt %s > %s", get_name().c_str(), - filename_c.c_str()); - assert(system(buff) != NULL); - FILE *fp = fopen(filename_c.c_str(), "r"); - fgets(buff, 1024, fp); - fclose(fp); - std::string fn(buff); - size_t pos1, pos2; - pos1 = fn.find_last_of("("); - pos2 = fn.find(")", pos1); - assert(pos2 > pos1 && pos1 > 0); - strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); - char *tok; - tok = strtok(buff, ","); - std::string tmp; - while (tok != NULL) { - std::string param(tok); - if (param.find("<") != std::string::npos) { - assert(param.find(">") == std::string::npos); - assert(param.find("*") == std::string::npos); - tmp = param; - } else { - if (tmp.length() > 0) { - tmp = ""; - assert(param.find(">") != std::string::npos); - assert(param.find("<") == std::string::npos); - assert(param.find("*") == std::string::npos); - } - printf("%s\n", param.c_str()); - if (param.find("*") != std::string::npos) { - paramIsPointer.push_back(true); - } else { - paramIsPointer.push_back(false); - } + //initialize paramList + char buff[1024]; + std::string filename_c(filename+"_c"); + snprintf(buff,1024,"c++filt %s > %s", get_name().c_str(), filename_c.c_str()); + assert(system(buff) != NULL); + FILE *fp = fopen(filename_c.c_str(), "r"); + fgets(buff, 1024, fp); + fclose(fp); + std::string fn(buff); + size_t pos1, pos2; + pos1 = fn.find_last_of("("); + pos2 = fn.find(")", pos1); + assert(pos2>pos1&&pos1>0); + strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); + char *tok; + tok = strtok(buff, ","); + std::string tmp; + while(tok!=NULL){ + std::string param(tok); + if(param.find("<")!=std::string::npos){ + assert(param.find(">")==std::string::npos); + assert(param.find("*")==std::string::npos); + tmp = param; + } else { + if (tmp.length()>0){ + tmp = ""; + assert(param.find(">")!=std::string::npos); + assert(param.find("<")==std::string::npos); + assert(param.find("*")==std::string::npos); + } + printf("%s\n", param.c_str()); + if(param.find("*")!=std::string::npos){ + paramIsPointer.push_back(true); + }else{ + paramIsPointer.push_back(false); + } + } + tok = strtok(NULL, ","); } - tok = strtok(NULL, ","); - } - for (std::map<unsigned, param_info>::iterator i = - m_ptx_kernel_param_info.begin(); - i != m_ptx_kernel_param_info.end(); i++) { - param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); - addr_t param_addr = param->get_address(); - param_t param_value = p.get_value(); - offsets.push_back((unsigned)p.get_offset()); - if (paramIsPointer[i->first] && - (*(unsigned long long *)param_value.pdata != 0)) { - // is pointer - assert(param_value.size == sizeof(void *) && - "MisID'd this param as pointer"); - size_t array_size = 0; - unsigned long long param_pointer = - *(unsigned long long *)param_value.pdata; - if (mallocPtr_Size.find(param_pointer) != mallocPtr_Size.end()) { - array_size = mallocPtr_Size[param_pointer]; - } else { - for (std::map<unsigned long long, size_t>::iterator j = - mallocPtr_Size.begin(); - j != mallocPtr_Size.end(); j++) { - if (param_pointer > j->first && - param_pointer < j->first + j->second) { - array_size = j->first + j->second - param_pointer; - break; - } - } - assert(array_size > 0 && "pointer was not previously malloc'd"); - } + for( std::map<unsigned,param_info>::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); + param_t param_value = p.get_value(); + offsets.push_back((unsigned)p.get_offset()); - unsigned char *val = (unsigned char *)malloc(param_value.size); - param_mem->read(param_addr, param_value.size, (void *)val); - unsigned char *array_val = (unsigned char *)malloc(array_size); - gpu->get_global_memory()->read(*(unsigned *)((void *)val), array_size, - (void *)array_val); - param_data.push_back( - std::pair<size_t, unsigned char *>(array_size, array_val)); - paramIsPointer.push_back(true); - } else { - unsigned char *val = (unsigned char *)malloc(param_value.size); - param_mem->read(param_addr, param_value.size, (void *)val); - param_data.push_back( - std::pair<size_t, unsigned char *>(param_value.size, val)); - paramIsPointer.push_back(false); + if (paramIsPointer[i->first] && (*(unsigned long long*)param_value.pdata != 0)){ + //is pointer + assert(param_value.size==sizeof(void*)&&"MisID'd this param as pointer"); + size_t array_size = 0; + unsigned long long param_pointer = *(unsigned long long*)param_value.pdata; + if(mallocPtr_Size.find(param_pointer)!=mallocPtr_Size.end()){ + array_size = mallocPtr_Size[param_pointer]; + }else{ + for( std::map<unsigned long long, size_t>::iterator j=mallocPtr_Size.begin(); j!=mallocPtr_Size.end(); j++ ) { + if(param_pointer>j->first&¶m_pointer<j->first + j->second){ + array_size = j->first + j->second - param_pointer; + break; + } + } + assert(array_size>0&&"pointer was not previously malloc'd"); + } + + unsigned char* val = (unsigned char*) malloc(param_value.size); + param_mem->read(param_addr,param_value.size,(void*)val); + unsigned char* array_val = (unsigned char*) malloc(array_size); + gpu->get_global_memory()->read(*(unsigned*)((void*)val),array_size,(void*)array_val); + param_data.push_back(std::pair<size_t, unsigned char*>(array_size,array_val)); + paramIsPointer.push_back(true); + }else{ + unsigned char* val = (unsigned char*) malloc(param_value.size); + param_mem->read(param_addr,param_value.size,(void*)val); + param_data.push_back(std::pair<size_t, unsigned char*>(param_value.size,val)); + paramIsPointer.push_back(false); + } } - } - FILE *fout = fopen(filename.c_str(), "w"); - printf("Writing data to %s ...\n", filename.c_str()); - fprintf(fout, "%s\n", get_name().c_str()); - fprintf(fout, "%u,%u,%u %u,%u,%u\n", gridDim.x, gridDim.y, gridDim.z, - blockDim.x, blockDim.y, blockDim.z); - size_t index = 0; - for (std::vector<std::pair<size_t, unsigned char *> >::const_iterator i = - param_data.begin(); - i != param_data.end(); i++) { - if (paramIsPointer[index]) { - fprintf(fout, "*"); + FILE *fout = fopen (filename.c_str(), "w"); + printf("Writing data to %s ...\n", filename.c_str()); + fprintf(fout, "%s\n", get_name().c_str()); + fprintf(fout, "%u,%u,%u %u,%u,%u\n", gridDim.x, gridDim.y, gridDim.z, blockDim.x, blockDim.y, blockDim.z); + size_t index = 0; + for( std::vector< std::pair<size_t,unsigned char*> >::const_iterator i=param_data.begin(); i!=param_data.end(); i++ ) { + if (paramIsPointer[index]){ + fprintf(fout, "*"); + } + fprintf(fout, "%lu :", i->first); + for (size_t j = 0; j<i->first; j++){ + fprintf(fout, " %u", i->second[j]); + } + fprintf(fout, " : %u", offsets[index]); + free (i->second); + fprintf(fout, "\n"); + index++; } - fprintf(fout, "%lu :", i->first); - for (size_t j = 0; j < i->first; j++) { - fprintf(fout, " %u", i->second[j]); + fflush(fout); + fclose(fout); + + //ptx config + std::string ptx_config_fn(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/ptx.config" + std::to_string(counter)); + snprintf(buff, 1024, "grep -rn \".entry %s\" %s/*.ptx | cut -d \":\" -f 1-2 > %s", get_name().c_str(), wys_exec_path, ptx_config_fn.c_str()); + if (system(buff)!=0){ + printf("WARNING: Failed to execute grep to find ptx source \n"); + printf("Problematic call: %s", buff); + abort(); } - fprintf(fout, " : %u", offsets[index]); - free(i->second); - fprintf(fout, "\n"); - index++; - } - fflush(fout); - fclose(fout); - - // ptx config - std::string ptx_config_fn(std::string(gpgpusim_path) + - "/debug_tools/WatchYourStep/data/ptx.config" + - std::to_string(counter)); - snprintf(buff, 1024, - "grep -rn \".entry %s\" %s/*.ptx | cut -d \":\" -f 1-2 > %s", - get_name().c_str(), wys_exec_path, ptx_config_fn.c_str()); - if (system(buff) != 0) { - printf("WARNING: Failed to execute grep to find ptx source \n"); - printf("Problematic call: %s", buff); - abort(); - } - FILE *fin = fopen(ptx_config_fn.c_str(), "r"); - char ptx_source[256]; - unsigned line_number; - int numscanned = fscanf(fin, "%[^:]:%u", ptx_source, &line_number); - assert(numscanned == 2); - fclose(fin); - snprintf(buff, 1024, - "grep -rn \".version\" %s | cut -d \":\" -f 1 | xargs -I \"{}\" awk " - "\"NR>={}&&NR<={}+2\" %s > %s", - ptx_source, ptx_source, ptx_config_fn.c_str()); - if (system(buff) != 0) { - printf("WARNING: Failed to execute grep to find ptx header \n"); - printf("Problematic call: %s", buff); - abort(); - } - fin = fopen(ptx_source, "r"); - assert(fin != NULL); - printf("Writing data to %s ...\n", ptx_config_fn.c_str()); - fout = fopen(ptx_config_fn.c_str(), "a"); - assert(fout != NULL); - for (unsigned i = 0; i < line_number; i++) { - assert(fgets(buff, 1024, fin) != NULL); - assert(!feof(fin)); - } - fprintf(fout, "\n\n"); - do { - fprintf(fout, "%s", buff); - assert(fgets(buff, 1024, fin) != NULL); - if (feof(fin)) { - break; + FILE *fin = fopen(ptx_config_fn.c_str(), "r"); + char ptx_source[256]; + unsigned line_number; + int numscanned = fscanf(fin, "%[^:]:%u", ptx_source, &line_number); + assert(numscanned == 2); + fclose(fin); + snprintf(buff, 1024, "grep -rn \".version\" %s | cut -d \":\" -f 1 | xargs -I \"{}\" awk \"NR>={}&&NR<={}+2\" %s > %s", ptx_source, ptx_source, ptx_config_fn.c_str()); + if (system(buff)!=0){ + printf("WARNING: Failed to execute grep to find ptx header \n"); + printf("Problematic call: %s", buff); + abort(); + } + fin = fopen(ptx_source, "r"); + assert(fin!=NULL); + printf("Writing data to %s ...\n", ptx_config_fn.c_str()); + fout = fopen(ptx_config_fn.c_str(), "a"); + assert(fout!=NULL); + for (unsigned i = 0; i<line_number; i++){ + assert(fgets(buff, 1024, fin) != NULL); + assert(!feof(fin)); } - } while (strstr(buff, "entry") == NULL); + fprintf(fout, "\n\n"); + do{ + fprintf(fout, "%s", buff); + assert(fgets(buff, 1024, fin) != NULL); + if(feof(fin)){ + break; + } + } while(strstr(buff, "entry")==NULL); - fclose(fin); - fflush(fout); - fclose(fout); - counter++; + fclose(fin); + fflush(fout); + fclose(fout); + counter++; } -template <int activate_level> -bool cuda_sim::ptx_debug_exec_dump_cond(int thd_uid, addr_t pc) { - if (g_debug_execution >= activate_level) { - // check each type of debug dump constraint to filter out dumps - if ((g_debug_thread_uid != 0) && - (thd_uid != (unsigned)g_debug_thread_uid)) { - return false; - } - if ((g_debug_pc != 0xBEEF1518) && (pc != g_debug_pc)) { - return false; - } - - return true; - } +template<int activate_level> +bool cuda_sim::ptx_debug_exec_dump_cond(int thd_uid, addr_t pc) +{ + if (g_debug_execution >= activate_level) { + // check each type of debug dump constraint to filter out dumps + if ( (g_debug_thread_uid != 0) && (thd_uid != (unsigned)g_debug_thread_uid) ) { + return false; + } + if ( (g_debug_pc != 0xBEEF1518) && (pc != g_debug_pc) ) { + return false; + } - return false; + return true; + } + + return false; } -void cuda_sim::init_inst_classification_stat() { - static std::set<unsigned> init; - if (init.find(g_ptx_kernel_count) != init.end()) return; - init.insert(g_ptx_kernel_count); +void cuda_sim::init_inst_classification_stat() +{ + static std::set<unsigned> init; + if( init.find(g_ptx_kernel_count) != init.end() ) + return; + init.insert(g_ptx_kernel_count); -#define MAX_CLASS_KER 1024 - char kernelname[MAX_CLASS_KER] = ""; - if (!g_inst_classification_stat) - g_inst_classification_stat = (void **)calloc(MAX_CLASS_KER, sizeof(void *)); - snprintf(kernelname, MAX_CLASS_KER, "Kernel %d Classification\n", - g_ptx_kernel_count); - assert(g_ptx_kernel_count < MAX_CLASS_KER); // a static limit on number of - // kernels increase it if it - // fails! - g_inst_classification_stat[g_ptx_kernel_count] = - StatCreate(kernelname, 1, 20); - if (!g_inst_op_classification_stat) - g_inst_op_classification_stat = - (void **)calloc(MAX_CLASS_KER, sizeof(void *)); - snprintf(kernelname, MAX_CLASS_KER, "Kernel %d OP Classification\n", - g_ptx_kernel_count); - g_inst_op_classification_stat[g_ptx_kernel_count] = - StatCreate(kernelname, 1, 100); + #define MAX_CLASS_KER 1024 + char kernelname[MAX_CLASS_KER] =""; + if (!g_inst_classification_stat) g_inst_classification_stat = (void**)calloc(MAX_CLASS_KER, sizeof(void*)); + snprintf(kernelname, MAX_CLASS_KER, "Kernel %d Classification\n",g_ptx_kernel_count ); + assert( g_ptx_kernel_count < MAX_CLASS_KER ) ; // a static limit on number of kernels increase it if it fails! + g_inst_classification_stat[g_ptx_kernel_count] = StatCreate(kernelname,1,20); + if (!g_inst_op_classification_stat) g_inst_op_classification_stat = (void**)calloc(MAX_CLASS_KER, sizeof(void*)); + snprintf(kernelname, MAX_CLASS_KER, "Kernel %d OP Classification\n",g_ptx_kernel_count ); + g_inst_op_classification_stat[g_ptx_kernel_count] = StatCreate(kernelname,1,100); } -static unsigned get_tex_datasize(const ptx_instruction *pI, - ptx_thread_info *thread) { - const operand_info &src1 = pI->src1(); // the name of the texture - std::string texname = src1.name(); +static unsigned get_tex_datasize( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &src1 = pI->src1(); //the name of the texture + std::string texname = src1.name(); - /* - For programs with many streams, textures can be bound and unbound - asynchronously. This means we need to use the kernel's "snapshot" of - the state of the texture mappings when it was launched (so that we - don't try to access the incorrect texture mapping if it's been updated, - or that we don't access a mapping that has been unbound). - */ - kernel_info_t &k = thread->get_kernel(); - const struct textureInfo *texInfo = k.get_texinfo(texname); + /* + For programs with many streams, textures can be bound and unbound + asynchronously. This means we need to use the kernel's "snapshot" of + the state of the texture mappings when it was launched (so that we + don't try to access the incorrect texture mapping if it's been updated, + or that we don't access a mapping that has been unbound). + */ + kernel_info_t& k = thread->get_kernel(); + const struct textureInfo* texInfo = k.get_texinfo(texname); - unsigned data_size = texInfo->texel_size; - return data_size; + unsigned data_size = texInfo->texel_size; + return data_size; } -int tensorcore_op(int inst_opcode) { - if ((inst_opcode == MMA_OP) || (inst_opcode == MMA_LD_OP) || - (inst_opcode == MMA_ST_OP)) - return 1; - else - return 0; +int tensorcore_op(int inst_opcode){ + + if((inst_opcode==MMA_OP)||(inst_opcode==MMA_LD_OP)||(inst_opcode==MMA_ST_OP)) + return 1; + else + return 0; } -void ptx_thread_info::ptx_exec_inst(warp_inst_t &inst, unsigned lane_id) { - bool skip = false; - int op_classification = 0; - addr_t pc = next_instr(); - assert(pc == - inst.pc); // make sure timing model and functional model are in sync - const ptx_instruction *pI = m_func_info->get_instruction(pc); +void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id) +{ + + bool skip = false; + int op_classification = 0; + addr_t pc = next_instr(); + assert( pc == inst.pc ); // make sure timing model and functional model are in sync + const ptx_instruction *pI = m_func_info->get_instruction(pc); + + set_npc( pc + pI->inst_size() ); + - set_npc(pc + pI->inst_size()); + try { - try { - clearRPC(); - m_last_set_operand_value.u64 = 0; + clearRPC(); + m_last_set_operand_value.u64 = 0; - if (is_done()) { - printf( - "attempted to execute instruction on a thread that is already " - "done.\n"); + if(is_done()) + { + printf("attempted to execute instruction on a thread that is already done.\n"); assert(0); - } - - if (g_debug_execution >= 6 || - m_gpu->get_config().get_ptx_inst_debug_to_file()) { - if ((m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid == 0) || - (get_uid() == - (unsigned)(m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid))) { - clear_modifiedregs(); - enable_debug_trace(); + } + + if ( g_debug_execution >= 6 || m_gpu->get_config().get_ptx_inst_debug_to_file()) { + if ( (m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid==0) + || (get_uid() == (unsigned)(m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid)) ) { + + clear_modifiedregs(); + enable_debug_trace(); } - } - - if (pI->has_pred()) { + } + + + if( pI->has_pred() ) { const operand_info &pred = pI->get_pred(); ptx_reg_t pred_value = get_operand_value(pred, pred, PRED_TYPE, this, 0); - if (pI->get_pred_mod() == -1) { - skip = (pred_value.pred & 0x0001) ^ - pI->get_pred_neg(); // ptxplus inverts the zero flag + if(pI->get_pred_mod() == -1) { + skip = (pred_value.pred & 0x0001) ^ pI->get_pred_neg(); //ptxplus inverts the zero flag } else { - skip = !pred_lookup(pI->get_pred_mod(), pred_value.pred & 0x000F); + skip = !pred_lookup(pI->get_pred_mod(), pred_value.pred & 0x000F); } - } - int inst_opcode = pI->get_opcode(); - - if (skip) { + } + int inst_opcode=pI->get_opcode(); + + if( skip ) { inst.set_not_active(lane_id); - } else { + } else { const ptx_instruction *pI_saved = pI; ptx_instruction *pJ = NULL; - if (pI->get_opcode() == VOTE_OP) { - pJ = new ptx_instruction(*pI); - *((warp_inst_t *)pJ) = inst; // copy active mask information - pI = pJ; + if( pI->get_opcode() == VOTE_OP ) { + pJ = new ptx_instruction(*pI); + *((warp_inst_t*)pJ) = inst; // copy active mask information + pI = pJ; } - - if (((inst_opcode == MMA_OP || inst_opcode == MMA_LD_OP || - inst_opcode == MMA_ST_OP))) { - if (inst.active_count() != MAX_WARP_SIZE) { - printf( - "Tensor Core operation are warp synchronous operation. All the " - "threads needs to be active."); - assert(0); - } + + if(((inst_opcode==MMA_OP||inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))){ + if(inst.active_count()!=MAX_WARP_SIZE) + { + printf("Tensor Core operation are warp synchronous operation. All the threads needs to be active."); + assert(0); + } } - - // Tensorcore is warp synchronous operation. So these instructions needs - // to be executed only once. To make the simulation faster removing the - // redundant tensorcore operation - if (!tensorcore_op(inst_opcode) || - ((tensorcore_op(inst_opcode)) && (lane_id == 0))) { - switch (inst_opcode) { -#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ - case OP: \ - FUNC(pI, this); \ - op_classification = CLASSIFICATION; \ - break; -#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ - case OP: \ - FUNC(pI, get_core(), inst); \ - op_classification = CLASSIFICATION; \ - break; -#include "opcodes.def" -#undef OP_DEF -#undef OP_W_DEF - default: - printf("Execution error: Invalid opcode (0x%x)\n", - pI->get_opcode()); - break; - } + + //Tensorcore is warp synchronous operation. So these instructions needs to be executed only once. To make the simulation faster removing the redundant tensorcore operation + if(!tensorcore_op(inst_opcode)||((tensorcore_op(inst_opcode))&&(lane_id==0))){ + switch ( inst_opcode ) { + #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,this); op_classification = CLASSIFICATION; break; + #define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,get_core(),inst); op_classification = CLASSIFICATION; break; + #include "opcodes.def" + #undef OP_DEF + #undef OP_W_DEF + default: printf( "Execution error: Invalid opcode (0x%x)\n", pI->get_opcode() ); break; + } } delete pJ; pI = pI_saved; - + // Run exit instruction if exit option included - if (pI->is_exit()) exit_impl(pI, this); - } + if(pI->is_exit()) + exit_impl(pI,this); + } + - const gpgpu_functional_sim_config &config = m_gpu->get_config(); - // Output instruction information to file and stdout - if (config.get_ptx_inst_debug_to_file() != 0 && - (config.get_ptx_inst_debug_thread_uid() == 0 || - config.get_ptx_inst_debug_thread_uid() == get_uid())) { - fprintf(m_gpu->get_ptx_inst_debug_file(), "[thd=%u] : (%s:%u - %s)\n", - get_uid(), pI->source_file(), pI->source_line(), - pI->get_source()); - // fprintf(ptx_inst_debug_file, "has memory read=%d, has memory - // write=%d\n", pI->has_memory_read(), pI->has_memory_write()); + const gpgpu_functional_sim_config &config = m_gpu->get_config(); + + // Output instruction information to file and stdout + if( config.get_ptx_inst_debug_to_file() != 0 && + (config.get_ptx_inst_debug_thread_uid() == 0 || config.get_ptx_inst_debug_thread_uid() == get_uid()) ) { + fprintf(m_gpu->get_ptx_inst_debug_file(), + "[thd=%u] : (%s:%u - %s)\n", + get_uid(), + pI->source_file(), pI->source_line(), pI->get_source() ); + //fprintf(ptx_inst_debug_file, "has memory read=%d, has memory write=%d\n", pI->has_memory_read(), pI->has_memory_write()); fflush(m_gpu->get_ptx_inst_debug_file()); - } + } - if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<5>(get_uid(), - pc)) { + if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<5>(get_uid(), pc) ) { dim3 ctaid = get_ctaid(); dim3 tid = get_tid(); - printf( - "%u [thd=%u][i=%u] : ctaid=(%u,%u,%u) tid=(%u,%u,%u) icount=%u " - "[pc=%u] (%s:%u - %s) [0x%llx]\n", - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, get_uid(), pI->uid(), - ctaid.x, ctaid.y, ctaid.z, tid.x, tid.y, tid.z, get_icount(), pc, - pI->source_file(), pI->source_line(), pI->get_source(), - m_last_set_operand_value.u64); + printf("%u [thd=%u][i=%u] : ctaid=(%u,%u,%u) tid=(%u,%u,%u) icount=%u [pc=%u] (%s:%u - %s) [0x%llx]\n", + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, + get_uid(), + pI->uid(), ctaid.x,ctaid.y,ctaid.z,tid.x,tid.y,tid.z, + get_icount(), + pc, pI->source_file(), pI->source_line(), pI->get_source(), + m_last_set_operand_value.u64 ); fflush(stdout); - } - - addr_t insn_memaddr = 0xFEEBDAED; - memory_space_t insn_space = undefined_space; - _memory_op_t insn_memory_op = no_memory_op; - unsigned insn_data_size = 0; - if ((pI->has_memory_read() || pI->has_memory_write())) { - if (!((inst_opcode == MMA_LD_OP || inst_opcode == MMA_ST_OP))) { + } + + addr_t insn_memaddr = 0xFEEBDAED; + memory_space_t insn_space = undefined_space; + _memory_op_t insn_memory_op = no_memory_op; + unsigned insn_data_size = 0; + if ( (pI->has_memory_read() || pI->has_memory_write()) ) { + if(!((inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))) + { insn_memaddr = last_eaddr(); insn_space = last_space(); unsigned to_type = pI->get_type(); insn_data_size = datatype2size(to_type); insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; - } - } - - if (pI->get_opcode() == BAR_OP && pI->barrier_op() == RED_OPTION) { - inst.add_callback(lane_id, last_callback().function, - last_callback().instruction, this, - false /*not atomic*/); - } + } + } + + if ( pI->get_opcode() == BAR_OP && pI->barrier_op() == RED_OPTION) { + inst.add_callback( lane_id, last_callback().function, last_callback().instruction, this,false /*not atomic*/); + } - if (pI->get_opcode() == ATOM_OP) { + if ( pI->get_opcode() == ATOM_OP ) { insn_memaddr = last_eaddr(); insn_space = last_space(); - inst.add_callback(lane_id, last_callback().function, - last_callback().instruction, this, true /*atomic*/); + inst.add_callback( lane_id, last_callback().function, last_callback().instruction, this,true /*atomic*/); unsigned to_type = pI->get_type(); insn_data_size = datatype2size(to_type); - } + } - if (pI->get_opcode() == TEX_OP) { - inst.set_addr(lane_id, last_eaddr()); - assert(inst.space == last_space()); - insn_data_size = get_tex_datasize( - pI, - this); // texture obtain its data granularity from the texture info - } + if (pI->get_opcode() == TEX_OP) { + inst.set_addr(lane_id, last_eaddr() ); + assert( inst.space == last_space() ); + insn_data_size = get_tex_datasize(pI, this); // texture obtain its data granularity from the texture info + } - // Output register information to file and stdout - if (config.get_ptx_inst_debug_to_file() != 0 && - (config.get_ptx_inst_debug_thread_uid() == 0 || - config.get_ptx_inst_debug_thread_uid() == get_uid())) { + // Output register information to file and stdout + if( config.get_ptx_inst_debug_to_file()!=0 && + (config.get_ptx_inst_debug_thread_uid()==0||config.get_ptx_inst_debug_thread_uid()==get_uid()) ) { dump_modifiedregs(m_gpu->get_ptx_inst_debug_file()); dump_regs(m_gpu->get_ptx_inst_debug_file()); - } - - if (g_debug_execution >= 6) { - if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<6>(get_uid(), - pc)) - dump_modifiedregs(stdout); - } - if (g_debug_execution >= 10) { - if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<10>(get_uid(), - pc)) - dump_regs(stdout); - } - update_pc(); - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn++; - - // not using it with functional simulation mode - if (!(this->m_functionalSimulationMode)) - ptx_file_line_stats_add_exec_count(pI); + } - if (m_gpu->gpgpu_ctx->func_sim->gpgpu_ptx_instruction_classification) { + if ( g_debug_execution >= 6 ) { + if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<6>(get_uid(), pc) ) + dump_modifiedregs(stdout); + } + if ( g_debug_execution >= 10 ) { + if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<10>(get_uid(), pc) ) + dump_regs(stdout); + } + update_pc(); + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn++; + + //not using it with functional simulation mode + if(!(this->m_functionalSimulationMode)) + ptx_file_line_stats_add_exec_count(pI); + + if ( m_gpu->gpgpu_ctx->func_sim->gpgpu_ptx_instruction_classification ) { m_gpu->gpgpu_ctx->func_sim->init_inst_classification_stat(); - unsigned space_type = 0; - switch (pI->get_space().get_type()) { - case global_space: - space_type = 10; - break; - case local_space: - space_type = 11; - break; - case tex_space: - space_type = 12; - break; - case surf_space: - space_type = 13; - break; - case param_space_kernel: - case param_space_local: - space_type = 14; - break; - case shared_space: - space_type = 15; - break; - case const_space: - space_type = 16; - break; - default: - space_type = 0; - break; + unsigned space_type=0; + switch ( pI->get_space().get_type() ) { + case global_space: space_type = 10; break; + case local_space: space_type = 11; break; + case tex_space: space_type = 12; break; + case surf_space: space_type = 13; break; + case param_space_kernel: + case param_space_local: + space_type = 14; break; + case shared_space: space_type = 15; break; + case const_space: space_type = 16; break; + default: + space_type = 0 ; + break; } - StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat - [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], - op_classification); - if (space_type) - StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat - [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], - (int)space_type); - StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_op_classification_stat - [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], - (int)pI->get_opcode()); - } - if ((m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn % 100000) == 0) { + StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], op_classification); + if (space_type) StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], ( int )space_type); + StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_op_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], (int) pI->get_opcode() ); + } + if ( (m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn % 100000) == 0 ) { dim3 ctaid = get_ctaid(); dim3 tid = get_tid(); - DPRINTF(LIVENESS, - "GPGPU-Sim PTX: %u instructions simulated : ctaid=(%u,%u,%u) " - "tid=(%u,%u,%u)\n", - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, ctaid.x, ctaid.y, - ctaid.z, tid.x, tid.y, tid.z); + DPRINTF(LIVENESS, "GPGPU-Sim PTX: %u instructions simulated : ctaid=(%u,%u,%u) tid=(%u,%u,%u)\n", + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, ctaid.x,ctaid.y,ctaid.z,tid.x,tid.y,tid.z ); fflush(stdout); - } - - // "Return values" - if (!skip) { - if (!((inst_opcode == MMA_LD_OP || inst_opcode == MMA_ST_OP))) { - inst.space = insn_space; - inst.set_addr(lane_id, insn_memaddr); - inst.data_size = insn_data_size; // simpleAtomicIntrinsics - assert(inst.memory_op == insn_memory_op); - } - } - - } catch (int x) { - printf("GPGPU-Sim PTX: ERROR (%d) executing intruction (%s:%u)\n", x, - pI->source_file(), pI->source_line()); - printf("GPGPU-Sim PTX: '%s'\n", pI->get_source()); - abort(); - } + } + + // "Return values" + if(!skip) { + if(!((inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))) + { + inst.space = insn_space; + inst.set_addr(lane_id, insn_memaddr); + inst.data_size = insn_data_size; // simpleAtomicIntrinsics + assert( inst.memory_op == insn_memory_op ); + } + } + + } catch ( int x ) { + printf("GPGPU-Sim PTX: ERROR (%d) executing intruction (%s:%u)\n", x, pI->source_file(), pI->source_line() ); + printf("GPGPU-Sim PTX: '%s'\n", pI->get_source() ); + abort(); + } + } -void cuda_sim::set_param_gpgpu_num_shaders(int num_shaders) { - gpgpu_param_num_shaders = num_shaders; +void cuda_sim::set_param_gpgpu_num_shaders(int num_shaders) +{ + gpgpu_param_num_shaders = num_shaders; } -const struct gpgpu_ptx_sim_info *ptx_sim_kernel_info( - const function_info *kernel) { - return kernel->get_kernel_info(); +const struct gpgpu_ptx_sim_info* ptx_sim_kernel_info(const function_info *kernel) +{ + return kernel->get_kernel_info(); } -const warp_inst_t *gpgpu_context::ptx_fetch_inst(address_type pc) { - return pc_to_instruction(pc); +const warp_inst_t *gpgpu_context::ptx_fetch_inst( address_type pc ) +{ + return pc_to_instruction(pc); } -unsigned ptx_sim_init_thread(kernel_info_t &kernel, - ptx_thread_info **thread_info, int sid, - unsigned tid, unsigned threads_left, - unsigned num_threads, core_t *core, - unsigned hw_cta_id, unsigned hw_warp_id, - gpgpu_t *gpu, bool isInFunctionalSimulationMode) { - std::list<ptx_thread_info *> &active_threads = kernel.active_threads(); +unsigned ptx_sim_init_thread( kernel_info_t &kernel, + ptx_thread_info** thread_info, + int sid, + unsigned tid, + unsigned threads_left, + unsigned num_threads, + core_t *core, + unsigned hw_cta_id, + unsigned hw_warp_id, + gpgpu_t *gpu, + bool isInFunctionalSimulationMode) +{ + std::list<ptx_thread_info *> &active_threads = kernel.active_threads(); - static std::map<unsigned, memory_space *> shared_memory_lookup; - static std::map<unsigned, memory_space *> sstarr_memory_lookup; - static std::map<unsigned, ptx_cta_info *> ptx_cta_lookup; - static std::map<unsigned, ptx_warp_info *> ptx_warp_lookup; - static std::map<unsigned, std::map<unsigned, memory_space *> > - local_memory_lookup; + static std::map<unsigned,memory_space*> shared_memory_lookup; + static std::map<unsigned,memory_space*> sstarr_memory_lookup; + static std::map<unsigned,ptx_cta_info*> ptx_cta_lookup; + static std::map<unsigned,ptx_warp_info*> ptx_warp_lookup; + static std::map<unsigned,std::map<unsigned,memory_space*> > local_memory_lookup; - if (*thread_info != NULL) { - ptx_thread_info *thd = *thread_info; - assert(thd->is_done()); - if (g_debug_execution == -1) { - dim3 ctaid = thd->get_ctaid(); - dim3 t = thd->get_tid(); - printf( - "GPGPU-Sim PTX simulator: thread exiting ctaid=(%u,%u,%u) " - "tid=(%u,%u,%u) uid=%u\n", - ctaid.x, ctaid.y, ctaid.z, t.x, t.y, t.z, thd->get_uid()); - fflush(stdout); - } - thd->m_cta_info->register_deleted_thread(thd); - delete thd; - *thread_info = NULL; - } + if ( *thread_info != NULL ) { + ptx_thread_info *thd = *thread_info; + assert( thd->is_done() ); + if ( g_debug_execution==-1 ) { + dim3 ctaid = thd->get_ctaid(); + dim3 t = thd->get_tid(); + printf("GPGPU-Sim PTX simulator: thread exiting ctaid=(%u,%u,%u) tid=(%u,%u,%u) uid=%u\n", + ctaid.x,ctaid.y,ctaid.z,t.x,t.y,t.z, thd->get_uid() ); + fflush(stdout); + } + thd->m_cta_info->register_deleted_thread(thd); + delete thd; + *thread_info = NULL; + } - if (!active_threads.empty()) { - assert(active_threads.size() <= threads_left); - ptx_thread_info *thd = active_threads.front(); - active_threads.pop_front(); - *thread_info = thd; - thd->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, - isInFunctionalSimulationMode); - return 1; - } + if ( !active_threads.empty() ) { + assert( active_threads.size() <= threads_left ); + ptx_thread_info *thd = active_threads.front(); + active_threads.pop_front(); + *thread_info = thd; + thd->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, isInFunctionalSimulationMode ); + return 1; + } - if (kernel.no_more_ctas_to_run()) { - return 0; // finished! - } + if ( kernel.no_more_ctas_to_run() ) { + return 0; //finished! + } - if (threads_left < kernel.threads_per_cta()) { - return 0; - } + if ( threads_left < kernel.threads_per_cta() ) { + return 0; + } - if (g_debug_execution == -1) { - printf("GPGPU-Sim PTX simulator: STARTING THREAD ALLOCATION --> \n"); - fflush(stdout); - } + if ( g_debug_execution==-1 ) { + printf("GPGPU-Sim PTX simulator: STARTING THREAD ALLOCATION --> \n"); + fflush(stdout); + } - // initializing new CTA - ptx_cta_info *cta_info = NULL; - memory_space *shared_mem = NULL; - memory_space *sstarr_mem = NULL; + //initializing new CTA + ptx_cta_info *cta_info = NULL; + memory_space *shared_mem = NULL; + memory_space *sstarr_mem = NULL; - unsigned cta_size = kernel.threads_per_cta(); - unsigned max_cta_per_sm = num_threads / cta_size; // e.g., 256 / 48 = 5 - assert(max_cta_per_sm > 0); + unsigned cta_size = kernel.threads_per_cta(); + unsigned max_cta_per_sm = num_threads/cta_size; // e.g., 256 / 48 = 5 + assert( max_cta_per_sm > 0 ); - // unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid; - unsigned sm_idx = - hw_cta_id * gpu->gpgpu_ctx->func_sim->gpgpu_param_num_shaders + sid; + //unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid; + unsigned sm_idx = hw_cta_id*gpu->gpgpu_ctx->func_sim->gpgpu_param_num_shaders + sid; - if (shared_memory_lookup.find(sm_idx) == shared_memory_lookup.end()) { - if (g_debug_execution >= 1) { - printf(" <CTA alloc> : sm_idx=%u sid=%u max_cta_per_sm=%u\n", sm_idx, - sid, max_cta_per_sm); - } - char buf[512]; - snprintf(buf, 512, "shared_%u", sid); - shared_mem = new memory_space_impl<16 * 1024>(buf, 4); - shared_memory_lookup[sm_idx] = shared_mem; - snprintf(buf, 512, "sstarr_%u", sid); - sstarr_mem = new memory_space_impl<16 * 1024>(buf, 4); - sstarr_memory_lookup[sm_idx] = sstarr_mem; - cta_info = new ptx_cta_info(sm_idx, gpu->gpgpu_ctx); - ptx_cta_lookup[sm_idx] = cta_info; - } else { - if (g_debug_execution >= 1) { - printf(" <CTA realloc> : sm_idx=%u sid=%u max_cta_per_sm=%u\n", sm_idx, - sid, max_cta_per_sm); - } - shared_mem = shared_memory_lookup[sm_idx]; - sstarr_mem = sstarr_memory_lookup[sm_idx]; - cta_info = ptx_cta_lookup[sm_idx]; - cta_info->check_cta_thread_status_and_reset(); - } + if ( shared_memory_lookup.find(sm_idx) == shared_memory_lookup.end() ) { + if ( g_debug_execution >= 1 ) { + printf(" <CTA alloc> : sm_idx=%u sid=%u max_cta_per_sm=%u\n", + sm_idx, sid, max_cta_per_sm ); + } + char buf[512]; + snprintf(buf,512,"shared_%u", sid); + shared_mem = new memory_space_impl<16*1024>(buf,4); + shared_memory_lookup[sm_idx] = shared_mem; + snprintf(buf,512,"sstarr_%u", sid); + sstarr_mem = new memory_space_impl<16*1024>(buf,4); + sstarr_memory_lookup[sm_idx] = sstarr_mem; + cta_info = new ptx_cta_info(sm_idx, gpu->gpgpu_ctx); + ptx_cta_lookup[sm_idx] = cta_info; + } else { + if ( g_debug_execution >= 1 ) { + printf(" <CTA realloc> : sm_idx=%u sid=%u max_cta_per_sm=%u\n", + sm_idx, sid, max_cta_per_sm ); + } + shared_mem = shared_memory_lookup[sm_idx]; + sstarr_mem = sstarr_memory_lookup[sm_idx]; + cta_info = ptx_cta_lookup[sm_idx]; + cta_info->check_cta_thread_status_and_reset(); + } - std::map<unsigned, memory_space *> &local_mem_lookup = - local_memory_lookup[sid]; - while (kernel.more_threads_in_cta()) { - dim3 ctaid3d = kernel.get_next_cta_id(); - unsigned new_tid = kernel.get_next_thread_id(); - dim3 tid3d = kernel.get_next_thread_id_3d(); - kernel.increment_thread_id(); - new_tid += tid; - ptx_thread_info *thd = new ptx_thread_info(kernel); - ptx_warp_info *warp_info = NULL; - if (ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end()) { - warp_info = new ptx_warp_info(); - ptx_warp_lookup[hw_warp_id] = warp_info; - } else { - warp_info = ptx_warp_lookup[hw_warp_id]; - } - thd->m_warp_info = warp_info; + std::map<unsigned,memory_space*> &local_mem_lookup = local_memory_lookup[sid]; + while( kernel.more_threads_in_cta() ) { + dim3 ctaid3d = kernel.get_next_cta_id(); + unsigned new_tid = kernel.get_next_thread_id(); + dim3 tid3d = kernel.get_next_thread_id_3d(); + kernel.increment_thread_id(); + new_tid += tid; + ptx_thread_info *thd = new ptx_thread_info(kernel); + ptx_warp_info *warp_info = NULL; + if ( ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end() ) { + warp_info = new ptx_warp_info(); + ptx_warp_lookup[hw_warp_id] = warp_info; + } else { + warp_info = ptx_warp_lookup[hw_warp_id]; + } + thd->m_warp_info = warp_info; - memory_space *local_mem = NULL; - std::map<unsigned, memory_space *>::iterator l = - local_mem_lookup.find(new_tid); - if (l != local_mem_lookup.end()) { - local_mem = l->second; - } else { - char buf[512]; - snprintf(buf, 512, "local_%u_%u", sid, new_tid); - local_mem = new memory_space_impl<32>(buf, 32); - local_mem_lookup[new_tid] = local_mem; - } - thd->set_info(kernel.entry()); - thd->set_nctaid(kernel.get_grid_dim()); - thd->set_ntid(kernel.get_cta_dim()); - thd->set_ctaid(ctaid3d); - thd->set_tid(tid3d); - if (kernel.entry()->get_ptx_version().extensions()) - thd->cpy_tid_to_reg(tid3d); - thd->set_valid(); - thd->m_shared_mem = shared_mem; - thd->m_sstarr_mem = sstarr_mem; - function_info *finfo = thd->func_info(); - symbol_table *st = finfo->get_symtab(); - thd->func_info()->param_to_shared(thd->m_shared_mem, st); - thd->func_info()->param_to_shared(thd->m_sstarr_mem, st); - thd->m_cta_info = cta_info; - cta_info->add_thread(thd); - thd->m_local_mem = local_mem; - if (g_debug_execution == -1) { - printf( - "GPGPU-Sim PTX simulator: allocating thread ctaid=(%u,%u,%u) " - "tid=(%u,%u,%u) @ 0x%Lx\n", - ctaid3d.x, ctaid3d.y, ctaid3d.z, tid3d.x, tid3d.y, tid3d.z, - (unsigned long long)thd); + memory_space *local_mem = NULL; + std::map<unsigned,memory_space*>::iterator l = local_mem_lookup.find(new_tid); + if ( l != local_mem_lookup.end() ) { + local_mem = l->second; + } else { + char buf[512]; + snprintf(buf,512,"local_%u_%u", sid, new_tid); + local_mem = new memory_space_impl<32>(buf,32); + local_mem_lookup[new_tid] = local_mem; + } + thd->set_info(kernel.entry()); + thd->set_nctaid(kernel.get_grid_dim()); + thd->set_ntid(kernel.get_cta_dim()); + thd->set_ctaid(ctaid3d); + thd->set_tid(tid3d); + if( kernel.entry()->get_ptx_version().extensions() ) + thd->cpy_tid_to_reg(tid3d); + thd->set_valid(); + thd->m_shared_mem = shared_mem; + thd->m_sstarr_mem = sstarr_mem; + function_info *finfo = thd->func_info(); + symbol_table *st = finfo->get_symtab(); + thd->func_info()->param_to_shared(thd->m_shared_mem,st); + thd->func_info()->param_to_shared(thd->m_sstarr_mem,st); + thd->m_cta_info = cta_info; + cta_info->add_thread(thd); + thd->m_local_mem = local_mem; + if ( g_debug_execution==-1 ) { + printf("GPGPU-Sim PTX simulator: allocating thread ctaid=(%u,%u,%u) tid=(%u,%u,%u) @ 0x%Lx\n", + ctaid3d.x,ctaid3d.y,ctaid3d.z,tid3d.x,tid3d.y,tid3d.z, (unsigned long long)thd ); + fflush(stdout); + } + active_threads.push_back(thd); + } + if ( g_debug_execution==-1 ) { + printf("GPGPU-Sim PTX simulator: <-- FINISHING THREAD ALLOCATION\n"); fflush(stdout); - } - active_threads.push_back(thd); - } - if (g_debug_execution == -1) { - printf("GPGPU-Sim PTX simulator: <-- FINISHING THREAD ALLOCATION\n"); - fflush(stdout); - } + } - kernel.increment_cta_id(); + kernel.increment_cta_id(); - assert(active_threads.size() <= threads_left); - *thread_info = active_threads.front(); - (*thread_info) - ->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, - isInFunctionalSimulationMode); - active_threads.pop_front(); - return 1; + assert( active_threads.size() <= threads_left ); + *thread_info = active_threads.front(); + (*thread_info)->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid,isInFunctionalSimulationMode ); + active_threads.pop_front(); + return 1; } -size_t get_kernel_code_size(class function_info *entry) { - return entry->get_function_size(); +size_t get_kernel_code_size( class function_info *entry ) +{ + return entry->get_function_size(); } -kernel_info_t *cuda_sim::gpgpu_opencl_ptx_sim_init_grid( - class function_info *entry, gpgpu_ptx_sim_arg_list_t args, - struct dim3 gridDim, struct dim3 blockDim, gpgpu_t *gpu) { - kernel_info_t *result = - new kernel_info_t(gridDim, blockDim, entry, gpu->getNameArrayMapping(), - gpu->getNameInfoMapping()); - unsigned argcount = args.size(); - unsigned argn = 1; - for (gpgpu_ptx_sim_arg_list_t::iterator a = args.begin(); a != args.end(); - a++) { - entry->add_param_data(argcount - argn, &(*a)); - argn++; - } - entry->finalize(result->get_param_memory()); - g_ptx_kernel_count++; - fflush(stdout); - return result; +kernel_info_t *cuda_sim::gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, + gpgpu_ptx_sim_arg_list_t args, + struct dim3 gridDim, + struct dim3 blockDim, + gpgpu_t *gpu ) +{ + kernel_info_t *result = new kernel_info_t(gridDim,blockDim,entry,gpu->getNameArrayMapping(),gpu->getNameInfoMapping()); + unsigned argcount=args.size(); + unsigned argn=1; + for( gpgpu_ptx_sim_arg_list_t::iterator a = args.begin(); a != args.end(); a++ ) { + entry->add_param_data(argcount-argn,&(*a)); + argn++; + } + entry->finalize(result->get_param_memory()); + g_ptx_kernel_count++; + fflush(stdout); + + return result; } #include "../../version" #include "detailed_version" -void print_splash() { - static int splash_printed = 0; - if (!splash_printed) { - fprintf(stdout, "\n\n *** %s [build %s] ***\n\n\n", - g_gpgpusim_version_string, g_gpgpusim_build_string); - splash_printed = 1; - } +void print_splash() +{ + static int splash_printed=0; + if ( !splash_printed ) { + fprintf(stdout, "\n\n *** %s [build %s] ***\n\n\n", g_gpgpusim_version_string, g_gpgpusim_build_string ); + splash_printed=1; + } } -void cuda_sim::gpgpu_ptx_sim_register_const_variable(void *hostVar, - const char *deviceName, - size_t size) { - printf("GPGPU-Sim PTX registering constant %s (%zu bytes) to name mapping\n", - deviceName, size); - g_const_name_lookup[hostVar] = deviceName; +void cuda_sim::gpgpu_ptx_sim_register_const_variable(void *hostVar, const char *deviceName, size_t size ) +{ + printf("GPGPU-Sim PTX registering constant %s (%zu bytes) to name mapping\n", deviceName, size ); + g_const_name_lookup[hostVar] = deviceName; } -void cuda_sim::gpgpu_ptx_sim_register_global_variable(void *hostVar, - const char *deviceName, - size_t size) { - printf("GPGPU-Sim PTX registering global %s hostVar to name mapping\n", - deviceName); - g_global_name_lookup[hostVar] = deviceName; +void cuda_sim::gpgpu_ptx_sim_register_global_variable(void *hostVar, const char *deviceName, size_t size ) +{ + printf("GPGPU-Sim PTX registering global %s hostVar to name mapping\n", deviceName ); + g_global_name_lookup[hostVar] = deviceName; } -void cuda_sim::gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, - size_t count, size_t offset, int to, - gpgpu_t *gpu) { - printf( - "GPGPU-Sim PTX: starting gpgpu_ptx_sim_memcpy_symbol with hostVar 0x%p\n", - hostVar); - bool found_sym = false; - memory_space_t mem_region = undefined_space; - std::string sym_name; +void cuda_sim::gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to, gpgpu_t *gpu ) +{ + printf("GPGPU-Sim PTX: starting gpgpu_ptx_sim_memcpy_symbol with hostVar 0x%p\n", hostVar); + bool found_sym = false; + memory_space_t mem_region = undefined_space; + std::string sym_name; - std::map<const void *, std::string>::iterator c = - gpu->gpgpu_ctx->func_sim->g_const_name_lookup.find(hostVar); - if (c != gpu->gpgpu_ctx->func_sim->g_const_name_lookup.end()) { - found_sym = true; - sym_name = c->second; - mem_region = const_space; - } - std::map<const void *, std::string>::iterator g = - gpu->gpgpu_ctx->func_sim->g_global_name_lookup.find(hostVar); - if (g != gpu->gpgpu_ctx->func_sim->g_global_name_lookup.end()) { - if (found_sym) { - printf( - "Execution error: PTX symbol \"%s\" w/ hostVar=0x%Lx is declared " - "both const and global?\n", - sym_name.c_str(), (unsigned long long)hostVar); - abort(); - } - found_sym = true; - sym_name = g->second; - mem_region = global_space; - } - if (g_globals.find(hostVar) != g_globals.end()) { - found_sym = true; - sym_name = hostVar; - mem_region = global_space; - } - if (g_constants.find(hostVar) != g_constants.end()) { - found_sym = true; - sym_name = hostVar; - mem_region = const_space; - } + std::map<const void*,std::string>::iterator c=gpu->gpgpu_ctx->func_sim->g_const_name_lookup.find(hostVar); + if ( c!=gpu->gpgpu_ctx->func_sim->g_const_name_lookup.end() ) { + found_sym = true; + sym_name = c->second; + mem_region = const_space; + } + std::map<const void*,std::string>::iterator g=gpu->gpgpu_ctx->func_sim->g_global_name_lookup.find(hostVar); + if ( g!=gpu->gpgpu_ctx->func_sim->g_global_name_lookup.end() ) { + if ( found_sym ) { + printf("Execution error: PTX symbol \"%s\" w/ hostVar=0x%Lx is declared both const and global?\n", + sym_name.c_str(), (unsigned long long)hostVar ); + abort(); + } + found_sym = true; + sym_name = g->second; + mem_region = global_space; + } + if( g_globals.find(hostVar) != g_globals.end() ) { + found_sym = true; + sym_name = hostVar; + mem_region = global_space; + } + if( g_constants.find(hostVar) != g_constants.end() ) { + found_sym = true; + sym_name = hostVar; + mem_region = const_space; + } - if (!found_sym) { - printf("Execution error: No information for PTX symbol w/ hostVar=0x%Lx\n", - (unsigned long long)hostVar); - abort(); - } else - printf( - "GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: Found PTX symbol w/ " - "hostVar=0x%Lx\n", - (unsigned long long)hostVar); - const char *mem_name = NULL; - memory_space *mem = NULL; + if ( !found_sym ) { + printf("Execution error: No information for PTX symbol w/ hostVar=0x%Lx\n", (unsigned long long)hostVar ); + abort(); + } else printf("GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: Found PTX symbol w/ hostVar=0x%Lx\n", (unsigned long long)hostVar ); + const char *mem_name = NULL; + memory_space *mem = NULL; - std::map<std::string, symbol_table *>::iterator st = - gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.find(sym_name.c_str()); - assert(st != gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.end()); - symbol_table *symtab = st->second; + std::map<std::string,symbol_table*>::iterator st = gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.find(sym_name.c_str()); + assert( st != gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.end() ); + symbol_table *symtab = st->second; - symbol *sym = symtab->lookup(sym_name.c_str()); - assert(sym); - unsigned dst = sym->get_address() + offset; - switch (mem_region.get_type()) { - case const_space: + symbol *sym = symtab->lookup(sym_name.c_str()); + assert(sym); + unsigned dst = sym->get_address() + offset; + switch (mem_region.get_type()) { + case const_space: mem = gpu->get_global_memory(); mem_name = "const"; break; - case global_space: + case global_space: mem = gpu->get_global_memory(); mem_name = "global"; break; - default: + default: abort(); - } - printf( - "GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: copying %s memory %zu bytes " - "%s symbol %s+%zu @0x%x ...\n", - mem_name, count, (to ? " to " : "from"), sym_name.c_str(), offset, dst); - for (unsigned n = 0; n < count; n++) { - if (to) - mem->write(dst + n, 1, ((char *)src) + n, NULL, NULL); - else - mem->read(dst + n, 1, ((char *)src) + n); - } - fflush(stdout); + } + printf("GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: copying %s memory %zu bytes %s symbol %s+%zu @0x%x ...\n", + mem_name, count, (to?" to ":"from"), sym_name.c_str(), offset, dst ); + for ( unsigned n=0; n < count; n++ ) { + if( to ) mem->write(dst+n,1,((char*)src)+n,NULL,NULL); + else mem->read(dst+n,1,((char*)src)+n); + } + fflush(stdout); } extern int ptx_debug; -void cuda_sim::read_sim_environment_variables() { - ptx_debug = 0; - g_debug_execution = 0; - g_interactive_debugger_enabled = false; +void cuda_sim::read_sim_environment_variables() +{ + ptx_debug = 0; + g_debug_execution = 0; + g_interactive_debugger_enabled = false; - char *mode = getenv("PTX_SIM_MODE_FUNC"); - if (mode) sscanf(mode, "%u", &g_ptx_sim_mode); - printf( - "GPGPU-Sim PTX: simulation mode %d (can change with PTX_SIM_MODE_FUNC " - "environment variable:\n", - g_ptx_sim_mode); - printf( - " 1=functional simulation only, 0=detailed performance " - "simulator)\n"); - char *dbg_inter = getenv("GPGPUSIM_DEBUG"); - if (dbg_inter && strlen(dbg_inter)) { - printf("GPGPU-Sim PTX: enabling interactive debugger\n"); - fflush(stdout); - g_interactive_debugger_enabled = true; - } - char *dbg_level = getenv("PTX_SIM_DEBUG"); - if (dbg_level && strlen(dbg_level)) { - printf("GPGPU-Sim PTX: setting debug level to %s\n", dbg_level); - fflush(stdout); - sscanf(dbg_level, "%d", &g_debug_execution); - } - char *dbg_thread = getenv("PTX_SIM_DEBUG_THREAD_UID"); - if (dbg_thread && strlen(dbg_thread)) { - printf("GPGPU-Sim PTX: printing debug information for thread uid %s\n", - dbg_thread); - fflush(stdout); - sscanf(dbg_thread, "%d", &g_debug_thread_uid); - } - char *dbg_pc = getenv("PTX_SIM_DEBUG_PC"); - if (dbg_pc && strlen(dbg_pc)) { - printf( - "GPGPU-Sim PTX: printing debug information for instruction with PC = " - "%s\n", - dbg_pc); - fflush(stdout); - sscanf(dbg_pc, "%d", &g_debug_pc); - } + char *mode = getenv("PTX_SIM_MODE_FUNC"); + if ( mode ) + sscanf(mode,"%u", &g_ptx_sim_mode); + printf("GPGPU-Sim PTX: simulation mode %d (can change with PTX_SIM_MODE_FUNC environment variable:\n", g_ptx_sim_mode); + printf(" 1=functional simulation only, 0=detailed performance simulator)\n"); + char *dbg_inter = getenv("GPGPUSIM_DEBUG"); + if ( dbg_inter && strlen(dbg_inter) ) { + printf("GPGPU-Sim PTX: enabling interactive debugger\n"); + fflush(stdout); + g_interactive_debugger_enabled = true; + } + char *dbg_level = getenv("PTX_SIM_DEBUG"); + if ( dbg_level && strlen(dbg_level) ) { + printf("GPGPU-Sim PTX: setting debug level to %s\n", dbg_level ); + fflush(stdout); + sscanf(dbg_level,"%d", &g_debug_execution); + } + char *dbg_thread = getenv("PTX_SIM_DEBUG_THREAD_UID"); + if ( dbg_thread && strlen(dbg_thread) ) { + printf("GPGPU-Sim PTX: printing debug information for thread uid %s\n", dbg_thread ); + fflush(stdout); + sscanf(dbg_thread,"%d", &g_debug_thread_uid); + } + char *dbg_pc = getenv("PTX_SIM_DEBUG_PC"); + if ( dbg_pc && strlen(dbg_pc) ) { + printf("GPGPU-Sim PTX: printing debug information for instruction with PC = %s\n", dbg_pc ); + fflush(stdout); + sscanf(dbg_pc,"%d", &g_debug_pc); + } #if CUDART_VERSION > 1010 - g_override_embedded_ptx = false; - char *usefile = getenv("PTX_SIM_USE_PTX_FILE"); - if (usefile && strlen(usefile)) { - printf( - "GPGPU-Sim PTX: overriding embedded ptx with ptx file " - "(PTX_SIM_USE_PTX_FILE is set)\n"); - fflush(stdout); - g_override_embedded_ptx = true; - } - char *blocking = getenv("CUDA_LAUNCH_BLOCKING"); - if (blocking && !strcmp(blocking, "1")) { - g_cuda_launch_blocking = true; - } + g_override_embedded_ptx = false; + char *usefile = getenv("PTX_SIM_USE_PTX_FILE"); + if (usefile && strlen(usefile)) { + printf("GPGPU-Sim PTX: overriding embedded ptx with ptx file (PTX_SIM_USE_PTX_FILE is set)\n"); + fflush(stdout); + g_override_embedded_ptx = true; + } + char *blocking = getenv("CUDA_LAUNCH_BLOCKING"); + if( blocking && !strcmp(blocking,"1") ) { + g_cuda_launch_blocking = true; + } #else - g_cuda_launch_blocking = true; - g_override_embedded_ptx = true; + g_cuda_launch_blocking = true; + g_override_embedded_ptx = true; #endif - if (g_debug_execution >= 40) { - ptx_debug = 1; - } + if ( g_debug_execution >= 40 ) { + ptx_debug = 1; + } } -#define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#define MAX(a,b) (((a)>(b))?(a):(b)) -unsigned max_cta(const struct gpgpu_ptx_sim_info *kernel_info, - unsigned threads_per_cta, unsigned int warp_size, - unsigned int n_thread_per_shader, - unsigned int gpgpu_shmem_size, - unsigned int gpgpu_shader_registers, - unsigned int max_cta_per_core) { - unsigned int padded_cta_size = threads_per_cta; - if (padded_cta_size % warp_size) - padded_cta_size = ((padded_cta_size / warp_size) + 1) * (warp_size); - unsigned int result_thread = n_thread_per_shader / padded_cta_size; +unsigned max_cta (const struct gpgpu_ptx_sim_info *kernel_info, unsigned threads_per_cta, unsigned int warp_size, unsigned int n_thread_per_shader, unsigned int gpgpu_shmem_size, unsigned int gpgpu_shader_registers, unsigned int max_cta_per_core) +{ + + unsigned int padded_cta_size = threads_per_cta; + if (padded_cta_size%warp_size) + padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size); + unsigned int result_thread = n_thread_per_shader / padded_cta_size; - unsigned int result_shmem = (unsigned)-1; - if (kernel_info->smem > 0) - result_shmem = gpgpu_shmem_size / kernel_info->smem; - unsigned int result_regs = (unsigned)-1; - if (kernel_info->regs > 0) - result_regs = gpgpu_shader_registers / - (padded_cta_size * ((kernel_info->regs + 3) & ~3)); - printf("padded cta size is %d and %d and %d", padded_cta_size, - kernel_info->regs, ((kernel_info->regs + 3) & ~3)); - // Limit by CTA - unsigned int result_cta = max_cta_per_core; + unsigned int result_shmem = (unsigned)-1; + if (kernel_info->smem > 0) + result_shmem = gpgpu_shmem_size / kernel_info->smem; + unsigned int result_regs = (unsigned)-1; + if (kernel_info->regs > 0) + result_regs = gpgpu_shader_registers / (padded_cta_size * ((kernel_info->regs+3)&~3)); + printf("padded cta size is %d and %d and %d",padded_cta_size, kernel_info->regs, ((kernel_info->regs+3)&~3) ); + //Limit by CTA + unsigned int result_cta = max_cta_per_core; - unsigned result = result_thread; - result = gs_min2(result, result_shmem); - result = gs_min2(result, result_regs); - result = gs_min2(result, result_cta); + unsigned result = result_thread; + result = gs_min2(result, result_shmem); + result = gs_min2(result, result_regs); + result = gs_min2(result, result_cta); - printf("GPGPU-Sim uArch: CTA/core = %u, limited by:", result); - if (result == result_thread) printf(" threads"); - if (result == result_shmem) printf(" shmem"); - if (result == result_regs) printf(" regs"); - if (result == result_cta) printf(" cta_limit"); - printf("\n"); + printf ("GPGPU-Sim uArch: CTA/core = %u, limited by:", result); + if (result == result_thread) printf (" threads"); + if (result == result_shmem) printf (" shmem"); + if (result == result_regs) printf (" regs"); + if (result == result_cta) printf (" cta_limit"); + printf ("\n"); - return result; + return result; } /*! -This function simulates the CUDA code functionally, it takes a kernel_info_t -parameter +This function simulates the CUDA code functionally, it takes a kernel_info_t parameter which holds the data for the CUDA kernel to be executed !*/ -void cuda_sim::gpgpu_cuda_ptx_sim_main_func(kernel_info_t &kernel, - bool openCL) { - printf( - "GPGPU-Sim: Performing Functional Simulation, executing kernel %s...\n", - kernel.name().c_str()); +void cuda_sim::gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) +{ + printf("GPGPU-Sim: Performing Functional Simulation, executing kernel %s...\n",kernel.name().c_str()); - // using a shader core object for book keeping, it is not needed but as most - // function built for performance simulation need it we use it here - // extern gpgpu_sim *g_the_gpu; - // before we execute, we should do PDOM analysis for functional simulation - // scenario. - function_info *kernel_func_info = kernel.entry(); - const struct gpgpu_ptx_sim_info *kernel_info = - ptx_sim_kernel_info(kernel_func_info); - checkpoint *g_checkpoint; - g_checkpoint = new checkpoint(); + //using a shader core object for book keeping, it is not needed but as most function built for performance simulation need it we use it here + //extern gpgpu_sim *g_the_gpu; + //before we execute, we should do PDOM analysis for functional simulation scenario. + function_info *kernel_func_info = kernel.entry(); + const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel_func_info); + checkpoint *g_checkpoint; + g_checkpoint = new checkpoint(); - if (kernel_func_info->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", - kernel.name().c_str()); - } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", - kernel.name().c_str()); - kernel_func_info->do_pdom(); - kernel_func_info->set_pdom(); - } + if (kernel_func_info->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", kernel.name().c_str() ); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", kernel.name().c_str() ); + kernel_func_info->do_pdom(); + kernel_func_info->set_pdom(); + } - unsigned max_cta_tot = max_cta( - kernel_info, kernel.threads_per_cta(), - gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size, - gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() - ->n_thread_per_shader, - gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() - ->gpgpu_shmem_size, - gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() - ->gpgpu_shader_registers, - gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() - ->max_cta_per_core); - printf("Max CTA : %d\n", max_cta_tot); + unsigned max_cta_tot = max_cta(kernel_info,kernel.threads_per_cta(), gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->n_thread_per_shader, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->gpgpu_shmem_size, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->gpgpu_shader_registers, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->max_cta_per_core); + printf("Max CTA : %d\n",max_cta_tot); - int cp_op = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_option; - int cp_kernel = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_kernel; - cp_count = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_insn_Y; - cp_cta_resume = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_CTA_t; - int cta_launched = 0; + int cp_op= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_option; + int cp_kernel= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_kernel; + cp_count= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_insn_Y; + cp_cta_resume= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_CTA_t; + int cta_launched =0; - // we excute the kernel one CTA (Block) at the time, as synchronization - // functions work block wise - while (!kernel.no_more_ctas_to_run()) { - unsigned temp = kernel.get_next_cta_id_single(); + //we excute the kernel one CTA (Block) at the time, as synchronization functions work block wise + while(!kernel.no_more_ctas_to_run()){ + unsigned temp=kernel.get_next_cta_id_single(); + - if (cp_op == 0 || (cp_op == 1 && cta_launched < cp_cta_resume && - kernel.get_uid() == cp_kernel) || - kernel.get_uid() < cp_kernel) // just fro testing - { - functionalCoreSim cta( - &kernel, gpgpu_ctx->the_gpgpusim->g_the_gpu, - gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size); - cta.execute(cp_count, temp); + if(cp_op==0 || (cp_op==1 && cta_launched<cp_cta_resume && kernel.get_uid()==cp_kernel) || kernel.get_uid()< cp_kernel) // just fro testing + { + functionalCoreSim cta( + &kernel, + gpgpu_ctx->the_gpgpusim->g_the_gpu, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size + ); + cta.execute(cp_count,temp); -#if (CUDART_VERSION >= 5000) - gpgpu_ctx->device_runtime->launch_all_device_kernels(); -#endif - } else { - kernel.increment_cta_id(); - } + #if (CUDART_VERSION >= 5000) + gpgpu_ctx->device_runtime->launch_all_device_kernels(); + #endif + } + else + { + kernel.increment_cta_id(); + } cta_launched++; - } + } - if (cp_op == 1) { - char f1name[2048]; - snprintf(f1name, 2048, "checkpoint_files/global_mem_%d.txt", - kernel.get_uid()); - g_checkpoint->store_global_mem( - gpgpu_ctx->the_gpgpusim->g_the_gpu->get_global_memory(), f1name, - (char *)"%08x"); - } + + + if(cp_op==1) + { + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", kernel.get_uid() ); + g_checkpoint->store_global_mem(gpgpu_ctx->the_gpgpusim->g_the_gpu->get_global_memory(), f1name , (char *)"%08x"); + } - // registering this kernel as done - // openCL kernel simulation calls don't register the kernel so we don't - // register its exit - if (!openCL) { - // extern stream_manager *g_stream_manager; - gpgpu_ctx->the_gpgpusim->g_stream_manager->register_finished_kernel( - kernel.get_uid()); - } + + + //registering this kernel as done + + //openCL kernel simulation calls don't register the kernel so we don't register its exit + if(!openCL) { + //extern stream_manager *g_stream_manager; + gpgpu_ctx->the_gpgpusim->g_stream_manager->register_finished_kernel(kernel.get_uid()); + } - //******PRINTING******* - printf("GPGPU-Sim: Done functional simulation (%u instructions simulated).\n", - g_ptx_sim_num_insn); - if (gpgpu_ptx_instruction_classification) { - StatDisp(g_inst_classification_stat[g_ptx_kernel_count]); - StatDisp(g_inst_op_classification_stat[g_ptx_kernel_count]); - } + //******PRINTING******* + printf( "GPGPU-Sim: Done functional simulation (%u instructions simulated).\n", g_ptx_sim_num_insn ); + if ( gpgpu_ptx_instruction_classification ) { + StatDisp( g_inst_classification_stat[g_ptx_kernel_count]); + StatDisp ( g_inst_op_classification_stat[g_ptx_kernel_count]); + } - // time_t variables used to calculate the total simulation time - // the start time of simulation is hold by the global variable - // g_simulation_starttime - // g_simulation_starttime is initilized by gpgpu_ptx_sim_init_perf() in - // gpgpusim_entrypoint.cc upon starting gpgpu-sim - time_t end_time, elapsed_time, days, hrs, minutes, sec; - end_time = time((time_t *)NULL); - elapsed_time = - MAX(end_time - gpgpu_ctx->the_gpgpusim->g_simulation_starttime, 1); + //time_t variables used to calculate the total simulation time + //the start time of simulation is hold by the global variable g_simulation_starttime + //g_simulation_starttime is initilized by gpgpu_ptx_sim_init_perf() in gpgpusim_entrypoint.cc upon starting gpgpu-sim + time_t end_time, elapsed_time, days, hrs, minutes, sec; + end_time = time((time_t *)NULL); + elapsed_time = MAX(end_time - gpgpu_ctx->the_gpgpusim->g_simulation_starttime, 1); + - // calculating and printing simulation time in terms of days, hours, minutes - // and seconds - days = elapsed_time / (3600 * 24); - hrs = elapsed_time / 3600 - 24 * days; - minutes = elapsed_time / 60 - 60 * (hrs + 24 * days); - sec = elapsed_time - 60 * (minutes + 60 * (hrs + 24 * days)); + //calculating and printing simulation time in terms of days, hours, minutes and seconds + days = elapsed_time/(3600*24); + hrs = elapsed_time/3600 - 24*days; + minutes = elapsed_time/60 - 60*(hrs + 24*days); + sec = elapsed_time - 60*(minutes + 60*(hrs + 24*days)); - fflush(stderr); - printf( - "\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", - (unsigned)days, (unsigned)hrs, (unsigned)minutes, (unsigned)sec, - (unsigned)elapsed_time); - printf("gpgpu_simulation_rate = %u (inst/sec)\n", - (unsigned)(g_ptx_sim_num_insn / elapsed_time)); - fflush(stdout); + fflush(stderr); + printf("\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", + (unsigned)days, (unsigned)hrs, (unsigned)minutes, (unsigned)sec, (unsigned)elapsed_time ); + printf("gpgpu_simulation_rate = %u (inst/sec)\n", (unsigned)(g_ptx_sim_num_insn / elapsed_time) ); + fflush(stdout); } -void functionalCoreSim::initializeCTA(unsigned ctaid_cp) { - int ctaLiveThreads = 0; - symbol_table *symtab = m_kernel->entry()->get_symtab(); - - for (int i = 0; i < m_warp_count; i++) { - m_warpAtBarrier[i] = false; - m_liveThreadCount[i] = 0; - } - for (int i = 0; i < m_warp_count * m_warp_size; i++) m_thread[i] = NULL; - - // get threads for a cta - for (unsigned i = 0; i < m_kernel->threads_per_cta(); i++) { - ptx_sim_init_thread(*m_kernel, &m_thread[i], 0, i, - m_kernel->threads_per_cta() - i, - m_kernel->threads_per_cta(), this, 0, i / m_warp_size, - (gpgpu_t *)m_gpu, true); - assert(m_thread[i] != NULL && !m_thread[i]->is_done()); - char fname[2048]; - snprintf(fname, 2048, "checkpoint_files/thread_%d_0_reg.txt", i); - if (m_gpu->gpgpu_ctx->func_sim->cp_cta_resume == 1) - m_thread[i]->resume_reg_thread(fname, symtab); - ctaLiveThreads++; - } +void functionalCoreSim::initializeCTA(unsigned ctaid_cp) +{ + int ctaLiveThreads=0; + symbol_table * symtab= m_kernel->entry()->get_symtab(); + + for(int i=0; i< m_warp_count; i++){ + m_warpAtBarrier[i]=false; + m_liveThreadCount[i]=0; + } + for(int i=0; i< m_warp_count*m_warp_size;i++) + m_thread[i]=NULL; + + //get threads for a cta + for(unsigned i=0; i<m_kernel->threads_per_cta();i++) { + ptx_sim_init_thread(*m_kernel,&m_thread[i],0,i,m_kernel->threads_per_cta()-i,m_kernel->threads_per_cta(),this,0,i/m_warp_size,(gpgpu_t*)m_gpu, true); + assert(m_thread[i]!=NULL && !m_thread[i]->is_done()); + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/thread_%d_0_reg.txt",i ); + if(m_gpu->gpgpu_ctx->func_sim->cp_cta_resume==1) + m_thread[i]->resume_reg_thread(fname,symtab); + ctaLiveThreads++; + } - for (int k = 0; k < m_warp_count; k++) createWarp(k); + for(int k=0;k<m_warp_count;k++) + createWarp(k); } -void functionalCoreSim::createWarp(unsigned warpId) { - simt_mask_t initialMask; - unsigned liveThreadsCount = 0; - initialMask.set(); - for (int i = warpId * m_warp_size; i < warpId * m_warp_size + m_warp_size; - i++) { - if (m_thread[i] == NULL) - initialMask.reset(i - warpId * m_warp_size); - else - liveThreadsCount++; - } +void functionalCoreSim::createWarp(unsigned warpId) +{ + simt_mask_t initialMask; + unsigned liveThreadsCount=0; + initialMask.set(); + for(int i=warpId*m_warp_size; i<warpId*m_warp_size+m_warp_size;i++){ + if(m_thread[i]==NULL) initialMask.reset(i-warpId*m_warp_size); + else liveThreadsCount++; + } + + assert(m_thread[warpId*m_warp_size]!=NULL); + m_simt_stack[warpId]->launch(m_thread[warpId*m_warp_size]->get_pc(),initialMask); + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/warp_%d_0_simt.txt",warpId ); - assert(m_thread[warpId * m_warp_size] != NULL); - m_simt_stack[warpId]->launch(m_thread[warpId * m_warp_size]->get_pc(), - initialMask); - char fname[2048]; - snprintf(fname, 2048, "checkpoint_files/warp_%d_0_simt.txt", warpId); + if(m_gpu->gpgpu_ctx->func_sim->cp_cta_resume==1) + { + unsigned pc,rpc; + m_simt_stack[warpId]->resume(fname); + m_simt_stack[warpId]->get_pdom_stack_top_info(&pc,&rpc); + for(int i=warpId*m_warp_size; i<warpId*m_warp_size+m_warp_size;i++){ + m_thread[i]->set_npc(pc); + m_thread[i]->update_pc(); + } - if (m_gpu->gpgpu_ctx->func_sim->cp_cta_resume == 1) { - unsigned pc, rpc; - m_simt_stack[warpId]->resume(fname); - m_simt_stack[warpId]->get_pdom_stack_top_info(&pc, &rpc); - for (int i = warpId * m_warp_size; i < warpId * m_warp_size + m_warp_size; - i++) { - m_thread[i]->set_npc(pc); - m_thread[i]->update_pc(); - } - } - m_liveThreadCount[warpId] = liveThreadsCount; + } + m_liveThreadCount[warpId]= liveThreadsCount; } -void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp) { - m_gpu->gpgpu_ctx->func_sim->cp_count = m_gpu->checkpoint_insn_Y; - m_gpu->gpgpu_ctx->func_sim->cp_cta_resume = m_gpu->checkpoint_CTA_t; - initializeCTA(ctaid_cp); - - int count = 0; - while (true) { - bool someOneLive = false; - bool allAtBarrier = true; - for (unsigned i = 0; i < m_warp_count; i++) { - executeWarp(i, allAtBarrier, someOneLive); - count++; - } - - if (inst_count > 0 && count > inst_count && - (m_kernel->get_uid() == m_gpu->checkpoint_kernel) && - (ctaid_cp >= m_gpu->checkpoint_CTA) && - (ctaid_cp < m_gpu->checkpoint_CTA_t) && m_gpu->checkpoint_option == 1) { - someOneLive = false; - break; - } - if (!someOneLive) break; - if (allAtBarrier) { - for (unsigned i = 0; i < m_warp_count; i++) m_warpAtBarrier[i] = false; +void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp) + { + m_gpu->gpgpu_ctx->func_sim->cp_count= m_gpu->checkpoint_insn_Y; + m_gpu->gpgpu_ctx->func_sim->cp_cta_resume= m_gpu->checkpoint_CTA_t; + initializeCTA(ctaid_cp); + + int count=0; + while(true){ + bool someOneLive= false; + bool allAtBarrier = true; + for(unsigned i=0;i<m_warp_count;i++){ + executeWarp(i,allAtBarrier,someOneLive); + count++; + } + + if(inst_count>0 && count>inst_count && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cp<m_gpu->checkpoint_CTA_t) && m_gpu->checkpoint_option==1) + { + someOneLive=false; + break; + } + if(!someOneLive) break; + if(allAtBarrier){ + for(unsigned i=0;i<m_warp_count;i++) + m_warpAtBarrier[i]=false; + } } - } - checkpoint *g_checkpoint; - g_checkpoint = new checkpoint(); + checkpoint *g_checkpoint; + g_checkpoint = new checkpoint(); + + ptx_reg_t regval; + regval.u64= 123; - ptx_reg_t regval; - regval.u64 = 123; - - unsigned ctaid = m_kernel->get_next_cta_id_single(); - if (m_gpu->checkpoint_option == 1 && - (m_kernel->get_uid() == m_gpu->checkpoint_kernel) && - (ctaid_cp >= m_gpu->checkpoint_CTA) && - (ctaid_cp < m_gpu->checkpoint_CTA_t)) { - char fname[2048]; - snprintf(fname, 2048, "checkpoint_files/shared_mem_%d.txt", ctaid - 1); - g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname, - (char *)"%08x"); - for (int i = 0; i < 32 * m_warp_count; i++) { - char fname[2048]; - snprintf(fname, 2048, "checkpoint_files/thread_%d_%d_reg.txt", i, - ctaid - 1); - m_thread[i]->print_reg_thread(fname); - char f1name[2048]; - snprintf(f1name, 2048, "checkpoint_files/local_mem_thread_%d_%d_reg.txt", - i, ctaid - 1); - g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name, - (char *)"%08x"); - m_thread[i]->set_done(); - m_thread[i]->exitCore(); - m_thread[i]->registerExit(); - } + unsigned ctaid =m_kernel->get_next_cta_id_single(); + if(m_gpu->checkpoint_option==1 && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cp<m_gpu->checkpoint_CTA_t)) + { + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/shared_mem_%d.txt",ctaid-1 ); + g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname , (char *)"%08x"); + for(int i=0; i<32*m_warp_count;i++) + { + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i,ctaid-1 ); + m_thread[i]->print_reg_thread(fname); + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i,ctaid-1 ); + g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name , (char *)"%08x"); + m_thread[i]->set_done(); + m_thread[i]->exitCore(); + m_thread[i]->registerExit(); + } + + for(int i=0;i<m_warp_count;i++) + { + + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/warp_%d_%d_simt.txt",i,ctaid-1 ); + FILE * fp = fopen(fname,"w"); + assert(fp!=NULL); + m_simt_stack[i]->print_checkpoint(fp); + fclose(fp); + } + } - for (int i = 0; i < m_warp_count; i++) { - char fname[2048]; - snprintf(fname, 2048, "checkpoint_files/warp_%d_%d_simt.txt", i, - ctaid - 1); - FILE *fp = fopen(fname, "w"); - assert(fp != NULL); - m_simt_stack[i]->print_checkpoint(fp); - fclose(fp); - } - } } -void functionalCoreSim::executeWarp(unsigned i, bool &allAtBarrier, - bool &someOneLive) { - if (!m_warpAtBarrier[i] && m_liveThreadCount[i] != 0) { - warp_inst_t inst = getExecuteWarp(i); - execute_warp_inst_t(inst, i); - if (inst.isatomic()) inst.do_atomic(true); - if (inst.op == BARRIER_OP || inst.op == MEMORY_BARRIER_OP) - m_warpAtBarrier[i] = true; - updateSIMTStack(i, &inst); - } - if (m_liveThreadCount[i] > 0) someOneLive = true; - if (!m_warpAtBarrier[i] && m_liveThreadCount[i] > 0) allAtBarrier = false; +void functionalCoreSim::executeWarp(unsigned i, bool &allAtBarrier, bool & someOneLive) +{ + if(!m_warpAtBarrier[i] && m_liveThreadCount[i]!=0){ + warp_inst_t inst =getExecuteWarp(i); + execute_warp_inst_t(inst,i); + if(inst.isatomic()) inst.do_atomic(true); + if(inst.op==BARRIER_OP || inst.op==MEMORY_BARRIER_OP ) m_warpAtBarrier[i]=true; + updateSIMTStack( i, &inst ); + } + if(m_liveThreadCount[i]>0) someOneLive=true; + if(!m_warpAtBarrier[i]&& m_liveThreadCount[i]>0) allAtBarrier = false; } -unsigned gpgpu_context::translate_pc_to_ptxlineno(unsigned pc) { - // this function assumes that the kernel fits inside a single PTX file - // function_info *pFunc = g_func_info; // assume that the current kernel is - // the one in query - const ptx_instruction *pInsn = pc_to_instruction(pc); - unsigned ptx_line_number = pInsn->source_line(); +unsigned gpgpu_context::translate_pc_to_ptxlineno(unsigned pc) +{ + // this function assumes that the kernel fits inside a single PTX file + // function_info *pFunc = g_func_info; // assume that the current kernel is the one in query + const ptx_instruction *pInsn = pc_to_instruction(pc); + unsigned ptx_line_number = pInsn->source_line(); - return ptx_line_number; + return ptx_line_number; } // ptxinfo parser -extern std::map<unsigned, const char *> get_duplicate(); +extern std::map<unsigned,const char*> get_duplicate(); static char *g_ptxinfo_kname = NULL; static struct gpgpu_ptx_sim_info g_ptxinfo; -static std::map<unsigned, const char *> g_duplicate; +static std::map<unsigned,const char*> g_duplicate; static const char *g_last_dup_type; -const char *get_ptxinfo_kname() { return g_ptxinfo_kname; } - -void print_ptxinfo() { - if (!get_ptxinfo_kname()) { - printf("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", g_ptxinfo.gmem, - g_ptxinfo.cmem); - } - if (get_ptxinfo_kname()) { - printf( - "GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", - get_ptxinfo_kname(), g_ptxinfo.regs, g_ptxinfo.lmem, g_ptxinfo.smem, - g_ptxinfo.cmem); - } +const char *get_ptxinfo_kname() +{ + return g_ptxinfo_kname; } -struct gpgpu_ptx_sim_info get_ptxinfo() { - return g_ptxinfo; +void print_ptxinfo() +{ + if(! get_ptxinfo_kname()){ + printf ("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", + g_ptxinfo.gmem, + g_ptxinfo.cmem); + } + if(get_ptxinfo_kname()){ + printf ("GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", + get_ptxinfo_kname(), + g_ptxinfo.regs, + g_ptxinfo.lmem, + g_ptxinfo.smem, + g_ptxinfo.cmem ); + } } -std::map<unsigned, const char *> get_duplicate() { return g_duplicate; } -void ptxinfo_linenum(unsigned linenum) { - g_duplicate[linenum] = g_last_dup_type; +struct gpgpu_ptx_sim_info get_ptxinfo() +{ + return g_ptxinfo; } -void ptxinfo_dup_type(const char *dup_type) { g_last_dup_type = dup_type; } +std::map<unsigned,const char*> get_duplicate() +{ + return g_duplicate; +} -void ptxinfo_function(const char *fname) { - clear_ptxinfo(); - g_ptxinfo_kname = strdup(fname); +void ptxinfo_linenum( unsigned linenum ) +{ + g_duplicate[linenum] = g_last_dup_type; } -void ptxinfo_regs(unsigned nregs) { g_ptxinfo.regs = nregs; } +void ptxinfo_dup_type( const char *dup_type ) +{ + g_last_dup_type = dup_type; +} -void ptxinfo_lmem(unsigned declared, unsigned system) { - g_ptxinfo.lmem = declared + system; +void ptxinfo_function(const char *fname ) +{ + clear_ptxinfo(); + g_ptxinfo_kname = strdup(fname); } -void ptxinfo_gmem(unsigned declared, unsigned system) { - g_ptxinfo.gmem = declared + system; +void ptxinfo_regs( unsigned nregs ) +{ + g_ptxinfo.regs=nregs; } -void ptxinfo_smem(unsigned declared, unsigned system) { - g_ptxinfo.smem = declared + system; +void ptxinfo_lmem( unsigned declared, unsigned system ) +{ + g_ptxinfo.lmem=declared+system; } -void ptxinfo_cmem(unsigned nbytes, unsigned bank) { g_ptxinfo.cmem += nbytes; } +void ptxinfo_gmem( unsigned declared, unsigned system ) +{ + g_ptxinfo.gmem=declared+system; +} -void clear_ptxinfo() { - free(g_ptxinfo_kname); - g_ptxinfo_kname = NULL; - g_ptxinfo.regs = 0; - g_ptxinfo.lmem = 0; - g_ptxinfo.smem = 0; - g_ptxinfo.cmem = 0; - g_ptxinfo.gmem = 0; - g_ptxinfo.ptx_version = 0; - g_ptxinfo.sm_target = 0; +void ptxinfo_smem( unsigned declared, unsigned system ) +{ + g_ptxinfo.smem=declared+system; } -void ptxinfo_opencl_addinfo(std::map<std::string, function_info *> &kernels) { - if (!g_ptxinfo_kname) { - printf("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", g_ptxinfo.gmem, - g_ptxinfo.cmem); - clear_ptxinfo(); - return; - } +void ptxinfo_cmem( unsigned nbytes, unsigned bank ) +{ + g_ptxinfo.cmem+=nbytes; +} - if (!strcmp("__cuda_dummy_entry__", g_ptxinfo_kname)) { - // this string produced by ptxas for empty ptx files (e.g., bandwidth test) - clear_ptxinfo(); - return; - } - std::map<std::string, function_info *>::iterator k = - kernels.find(g_ptxinfo_kname); - if (k == kernels.end()) { - printf("GPGPU-Sim PTX: ERROR ** implementation for '%s' not found.\n", - g_ptxinfo_kname); - abort(); - } else { - printf( - "GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", - g_ptxinfo_kname, g_ptxinfo.regs, g_ptxinfo.lmem, g_ptxinfo.smem, - g_ptxinfo.cmem); - function_info *finfo = k->second; - assert(finfo != NULL); - finfo->set_kernel_info(g_ptxinfo); - } - clear_ptxinfo(); +void clear_ptxinfo() +{ + free(g_ptxinfo_kname); + g_ptxinfo_kname=NULL; + g_ptxinfo.regs=0; + g_ptxinfo.lmem=0; + g_ptxinfo.smem=0; + g_ptxinfo.cmem=0; + g_ptxinfo.gmem=0; + g_ptxinfo.ptx_version=0; + g_ptxinfo.sm_target=0; } -struct rec_pts cuda_sim::find_reconvergence_points(function_info *finfo) { - rec_pts tmp; - std::map<function_info *, rec_pts>::iterator r = g_rpts.find(finfo); - if (r == g_rpts.end()) { - int num_recon = finfo->get_num_reconvergence_pairs(); +void ptxinfo_opencl_addinfo( std::map<std::string,function_info*> &kernels ) +{ - gpgpu_recon_t *kernel_recon_points = - (struct gpgpu_recon_t *)calloc(num_recon, sizeof(struct gpgpu_recon_t)); - finfo->get_reconvergence_pairs(kernel_recon_points); - printf("GPGPU-Sim PTX: reconvergence points for %s...\n", - finfo->get_name().c_str()); - for (int i = 0; i < num_recon; i++) { - printf("GPGPU-Sim PTX: %2u (potential) branch divergence @ ", i + 1); - kernel_recon_points[i].source_inst->print_insn(); - printf("\n"); - printf("GPGPU-Sim PTX: immediate post dominator @ "); - if (kernel_recon_points[i].target_inst) - kernel_recon_points[i].target_inst->print_insn(); - printf("\n"); - } - printf("GPGPU-Sim PTX: ... end of reconvergence points for %s\n", - finfo->get_name().c_str()); + if(! g_ptxinfo_kname) { + printf ("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", + g_ptxinfo.gmem, + g_ptxinfo.cmem); + clear_ptxinfo(); + return; + } + + if( !strcmp("__cuda_dummy_entry__",g_ptxinfo_kname) ) { + // this string produced by ptxas for empty ptx files (e.g., bandwidth test) + clear_ptxinfo(); + return; + } + std::map<std::string,function_info*>::iterator k=kernels.find(g_ptxinfo_kname); + if( k==kernels.end() ) { + printf ("GPGPU-Sim PTX: ERROR ** implementation for '%s' not found.\n", g_ptxinfo_kname ); + abort(); + } else { + printf ("GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", + g_ptxinfo_kname, + g_ptxinfo.regs, + g_ptxinfo.lmem, + g_ptxinfo.smem, + g_ptxinfo.cmem ); + function_info *finfo = k->second; + assert(finfo!=NULL); + finfo->set_kernel_info( g_ptxinfo ); + } + clear_ptxinfo(); +} + +struct rec_pts cuda_sim::find_reconvergence_points( function_info *finfo ) +{ + rec_pts tmp; + std::map<function_info*,rec_pts>::iterator r=g_rpts.find(finfo); + + if( r==g_rpts.end() ) { + int num_recon = finfo->get_num_reconvergence_pairs(); + + gpgpu_recon_t *kernel_recon_points = (struct gpgpu_recon_t*) calloc(num_recon, sizeof(struct gpgpu_recon_t)); + finfo->get_reconvergence_pairs(kernel_recon_points); + printf("GPGPU-Sim PTX: reconvergence points for %s...\n", finfo->get_name().c_str() ); + for (int i=0;i<num_recon;i++) { + printf("GPGPU-Sim PTX: %2u (potential) branch divergence @ ", i+1 ); + kernel_recon_points[i].source_inst->print_insn(); + printf("\n"); + printf("GPGPU-Sim PTX: immediate post dominator @ " ); + if( kernel_recon_points[i].target_inst ) + kernel_recon_points[i].target_inst->print_insn(); + printf("\n"); + } + printf("GPGPU-Sim PTX: ... end of reconvergence points for %s\n", finfo->get_name().c_str() ); - tmp.s_kernel_recon_points = kernel_recon_points; - tmp.s_num_recon = num_recon; - g_rpts[finfo] = tmp; - } else { - tmp = r->second; - } - return tmp; + tmp.s_kernel_recon_points = kernel_recon_points; + tmp.s_num_recon = num_recon; + g_rpts[finfo] = tmp; + } else { + tmp = r->second; + } + return tmp; } -address_type get_return_pc(void *thd) { - // function call return - ptx_thread_info *the_thread = (ptx_thread_info *)thd; - assert(the_thread != NULL); - return the_thread->get_return_PC(); +address_type get_return_pc( void *thd ) +{ + // function call return + ptx_thread_info *the_thread = (ptx_thread_info*)thd; + assert( the_thread != NULL ); + return the_thread->get_return_PC(); } -address_type cuda_sim::get_converge_point(address_type pc) { - // the branch could encode the reconvergence point and/or a bit that indicates - // the - // reconvergence point is the return PC on the call stack in the case the - // branch has - // no immediate postdominator in the function (i.e., due to multiple return - // points). +address_type cuda_sim::get_converge_point( address_type pc ) +{ + // the branch could encode the reconvergence point and/or a bit that indicates the + // reconvergence point is the return PC on the call stack in the case the branch has + // no immediate postdominator in the function (i.e., due to multiple return points). - std::map<unsigned, function_info *>::iterator f = g_pc_to_finfo.find(pc); - assert(f != g_pc_to_finfo.end()); - function_info *finfo = f->second; - rec_pts tmp = find_reconvergence_points(finfo); + std::map<unsigned,function_info*>::iterator f=g_pc_to_finfo.find(pc); + assert( f != g_pc_to_finfo.end() ); + function_info *finfo = f->second; + rec_pts tmp = find_reconvergence_points(finfo); - int i = 0; - for (; i < tmp.s_num_recon; ++i) { - if (tmp.s_kernel_recon_points[i].source_pc == pc) { - if (tmp.s_kernel_recon_points[i].target_pc == (unsigned)-2) { - return RECONVERGE_RETURN_PC; - } else { - return tmp.s_kernel_recon_points[i].target_pc; + int i=0; + for (; i < tmp.s_num_recon; ++i) { + if (tmp.s_kernel_recon_points[i].source_pc == pc) { + if( tmp.s_kernel_recon_points[i].target_pc == (unsigned) -2 ) { + return RECONVERGE_RETURN_PC; + } else { + return tmp.s_kernel_recon_points[i].target_pc; + } } - } - } - return NO_BRANCH_DIVERGENCE; + } + return NO_BRANCH_DIVERGENCE; } -void functionalCoreSim::warp_exit(unsigned warp_id) { - for (int i = 0; i < m_warp_count * m_warp_size; i++) { - if (m_thread[i] != NULL) { - m_thread[i]->m_cta_info->register_deleted_thread(m_thread[i]); - delete m_thread[i]; +void functionalCoreSim::warp_exit( unsigned warp_id ) +{ + for(int i=0;i<m_warp_count*m_warp_size;i++){ + if(m_thread[i]!=NULL){ + m_thread[i]->m_cta_info->register_deleted_thread(m_thread[i]); + delete m_thread[i]; + } } - } } diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h index b828749..1be3d19 100644 --- a/src/cuda-sim/cuda-sim.h +++ b/src/cuda-sim/cuda-sim.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -30,13 +28,13 @@ #ifndef CUDASIM_H_INCLUDED #define CUDASIM_H_INCLUDED +#include "../abstract_hardware_model.h" +#include"../gpgpu-sim/shader.h" #include <stdlib.h> #include <map> -#include <string> #include <vector> -#include "../abstract_hardware_model.h" -#include "../gpgpu-sim/shader.h" -#include "ptx_sim.h" +#include <string> +#include"ptx_sim.h" class gpgpu_context; class memory_space; @@ -46,65 +44,69 @@ class symbol_table; extern const char *g_gpgpusim_version_string; extern int g_debug_execution; -extern void print_splash(); +extern void print_splash(); -extern void ptxinfo_opencl_addinfo( - std::map<std::string, function_info *> &kernels); -unsigned ptx_sim_init_thread(kernel_info_t &kernel, - class ptx_thread_info **thread_info, int sid, - unsigned tid, unsigned threads_left, - unsigned num_threads, class core_t *core, - unsigned hw_cta_id, unsigned hw_warp_id, - gpgpu_t *gpu, - bool functionalSimulationMode = false); -const struct gpgpu_ptx_sim_info *ptx_sim_kernel_info( - const class function_info *kernel); +extern void ptxinfo_opencl_addinfo( std::map<std::string,function_info*> &kernels ); +unsigned ptx_sim_init_thread( kernel_info_t &kernel, + class ptx_thread_info** thread_info, + int sid, + unsigned tid, + unsigned threads_left, + unsigned num_threads, + class core_t *core, + unsigned hw_cta_id, + unsigned hw_warp_id, + gpgpu_t *gpu, + bool functionalSimulationMode = false); +const struct gpgpu_ptx_sim_info* ptx_sim_kernel_info(const class function_info *kernel); /*! - * This class functionally executes a kernel. It uses the basic data structures - * and procedures in core_t + * This class functionally executes a kernel. It uses the basic data structures and procedures in core_t */ -class functionalCoreSim : public core_t { - public: - functionalCoreSim(kernel_info_t *kernel, gpgpu_sim *g, unsigned warp_size) - : core_t(g, kernel, warp_size, kernel->threads_per_cta()) { - m_warpAtBarrier = new bool[m_warp_count]; - m_liveThreadCount = new unsigned[m_warp_count]; - } - virtual ~functionalCoreSim() { - warp_exit(0); - delete[] m_liveThreadCount; - delete[] m_warpAtBarrier; - } - //! executes all warps till completion - void execute(int inst_count, unsigned ctaid_cp); - virtual void warp_exit(unsigned warp_id); - virtual bool warp_waiting_at_barrier(unsigned warp_id) const { - return (m_warpAtBarrier[warp_id] || !(m_liveThreadCount[warp_id] > 0)); - } - - private: - void executeWarp(unsigned, bool &, bool &); - // initializes threads in the CTA block which we are executing - void initializeCTA(unsigned ctaid_cp); - virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, - unsigned tid) { - if (m_thread[tid] == NULL || m_thread[tid]->is_done()) { - m_liveThreadCount[tid / m_warp_size]--; +class functionalCoreSim: public core_t +{ +public: + functionalCoreSim(kernel_info_t * kernel, gpgpu_sim *g, unsigned warp_size) + : core_t( g, kernel, warp_size, kernel->threads_per_cta() ) + { + m_warpAtBarrier = new bool [m_warp_count]; + m_liveThreadCount = new unsigned [m_warp_count]; } - } - - // lunches the stack and set the threads count - void createWarp(unsigned warpId); - - // each warp live thread count and barrier indicator - unsigned *m_liveThreadCount; - bool *m_warpAtBarrier; + virtual ~functionalCoreSim(){ + warp_exit(0); + delete[] m_liveThreadCount; + delete[] m_warpAtBarrier; + } + //! executes all warps till completion + void execute(int inst_count, unsigned ctaid_cp); + virtual void warp_exit( unsigned warp_id ); + virtual bool warp_waiting_at_barrier( unsigned warp_id ) const + { + return (m_warpAtBarrier[warp_id] || !(m_liveThreadCount[warp_id]>0)); + } + +private: + void executeWarp(unsigned, bool &, bool &); + //initializes threads in the CTA block which we are executing + void initializeCTA(unsigned ctaid_cp); + virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid) + { + if(m_thread[tid]==NULL || m_thread[tid]->is_done()){ + m_liveThreadCount[tid/m_warp_size]--; + } + } + + // lunches the stack and set the threads count + void createWarp(unsigned warpId); + + //each warp live thread count and barrier indicator + unsigned * m_liveThreadCount; + bool* m_warpAtBarrier; }; #define RECONVERGE_RETURN_PC ((address_type)-2) #define NO_BRANCH_DIVERGENCE ((address_type)-1) -address_type get_return_pc(void *thd); +address_type get_return_pc( void *thd ); const char *get_ptxinfo_kname(); void print_ptxinfo(); void clear_ptxinfo(); @@ -112,98 +114,89 @@ struct gpgpu_ptx_sim_info get_ptxinfo(); class gpgpu_recon_t; struct rec_pts { - gpgpu_recon_t *s_kernel_recon_points; - int s_num_recon; + gpgpu_recon_t *s_kernel_recon_points; + int s_num_recon; }; + class cuda_sim { - public: - cuda_sim(gpgpu_context *ctx) { - g_ptx_sim_num_insn = 0; - g_ptx_kernel_count = - -1; // used for classification stat collection purposes - gpgpu_param_num_shaders = 0; - g_cuda_launch_blocking = false; - g_inst_classification_stat = NULL; - g_inst_op_classification_stat = NULL; - g_assemble_code_next_pc = 0; - g_debug_thread_uid = 0; - g_override_embedded_ptx = false; - ptx_tex_regs = NULL; - g_ptx_thread_info_delete_count = 0; - g_ptx_thread_info_uid_next = 1; - g_debug_pc = 0xBEEF1518; - gpgpu_ctx = ctx; - } - // global variables - char *opcode_latency_int; - char *opcode_latency_fp; - char *opcode_latency_dp; - char *opcode_latency_sfu; - char *opcode_latency_tensor; - char *opcode_initiation_int; - char *opcode_initiation_fp; - char *opcode_initiation_dp; - char *opcode_initiation_sfu; - char *opcode_initiation_tensor; - int cp_count; - int cp_cta_resume; - int g_ptxinfo_error_detected; - unsigned g_ptx_sim_num_insn; - char *cdp_latency_str; - int g_ptx_kernel_count; // used for classification stat collection purposes - std::map<const void *, std::string> - g_global_name_lookup; // indexed by hostVar - std::map<const void *, std::string> - g_const_name_lookup; // indexed by hostVar - int g_ptx_sim_mode; // if non-zero run functional simulation only (i.e., no - // notion of a clock cycle) - unsigned gpgpu_param_num_shaders; - class std::map<function_info *, rec_pts> g_rpts; - bool g_cuda_launch_blocking; - void **g_inst_classification_stat; - void **g_inst_op_classification_stat; - std::set<std::string> g_globals; - std::set<std::string> g_constants; - std::map<unsigned, function_info *> g_pc_to_finfo; - int gpgpu_ptx_instruction_classification; - unsigned cdp_latency[5]; - unsigned g_assemble_code_next_pc; - int g_debug_thread_uid; - bool g_override_embedded_ptx; - std::set<unsigned long long> g_ptx_cta_info_sm_idx_used; - ptx_reg_t *ptx_tex_regs; - unsigned g_ptx_thread_info_delete_count; - unsigned g_ptx_thread_info_uid_next; - addr_t g_debug_pc; - // backward pointer - class gpgpu_context *gpgpu_ctx; - // global functions - void ptx_opcocde_latency_options(option_parser_t opp); - void gpgpu_cuda_ptx_sim_main_func(kernel_info_t &kernel, bool openCL = false); - int gpgpu_opencl_ptx_sim_main_func(kernel_info_t *grid); - void init_inst_classification_stat(); - kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, - gpgpu_ptx_sim_arg_list_t args, - struct dim3 gridDim, - struct dim3 blockDim, - gpgpu_t *gpu); - void gpgpu_ptx_sim_register_global_variable(void *hostVar, - const char *deviceName, - size_t size); - void gpgpu_ptx_sim_register_const_variable(void *, const char *deviceName, - size_t size); - void read_sim_environment_variables(); - void set_param_gpgpu_num_shaders(int num_shaders); - struct rec_pts find_reconvergence_points(function_info *finfo); - address_type get_converge_point(address_type pc); - void gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, - size_t count, size_t offset, int to, - gpgpu_t *gpu); - void ptx_print_insn(address_type pc, FILE *fp); - std::string ptx_get_insn_str(address_type pc); - template <int activate_level> - bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc); + public: + cuda_sim( gpgpu_context* ctx ) { + g_ptx_sim_num_insn = 0; + g_ptx_kernel_count = -1; // used for classification stat collection purposes + gpgpu_param_num_shaders = 0; + g_cuda_launch_blocking = false; + g_inst_classification_stat = NULL; + g_inst_op_classification_stat= NULL; + g_assemble_code_next_pc=0; + g_debug_thread_uid = 0; + g_override_embedded_ptx = false; + ptx_tex_regs = NULL; + g_ptx_thread_info_delete_count=0; + g_ptx_thread_info_uid_next=1; + g_debug_pc = 0xBEEF1518; + gpgpu_ctx = ctx; + } + //global variables + char *opcode_latency_int; + char *opcode_latency_fp; + char *opcode_latency_dp; + char *opcode_latency_sfu; + char *opcode_latency_tensor; + char *opcode_initiation_int; + char *opcode_initiation_fp; + char *opcode_initiation_dp; + char *opcode_initiation_sfu; + char *opcode_initiation_tensor; + int cp_count; + int cp_cta_resume; + int g_ptxinfo_error_detected; + unsigned g_ptx_sim_num_insn; + char *cdp_latency_str; + int g_ptx_kernel_count; // used for classification stat collection purposes + std::map<const void*,std::string> g_global_name_lookup; // indexed by hostVar + std::map<const void*,std::string> g_const_name_lookup; // indexed by hostVar + int g_ptx_sim_mode; // if non-zero run functional simulation only (i.e., no notion of a clock cycle) + unsigned gpgpu_param_num_shaders; + class std::map<function_info*,rec_pts> g_rpts; + bool g_cuda_launch_blocking; + void ** g_inst_classification_stat; + void ** g_inst_op_classification_stat; + std::set<std::string> g_globals; + std::set<std::string> g_constants; + std::map<unsigned,function_info*> g_pc_to_finfo; + int gpgpu_ptx_instruction_classification; + unsigned cdp_latency[5]; + unsigned g_assemble_code_next_pc; + int g_debug_thread_uid; + bool g_override_embedded_ptx; + std::set<unsigned long long> g_ptx_cta_info_sm_idx_used; + ptx_reg_t* ptx_tex_regs; + unsigned g_ptx_thread_info_delete_count; + unsigned g_ptx_thread_info_uid_next; + addr_t g_debug_pc; + // backward pointer + class gpgpu_context* gpgpu_ctx; + //global functions + void ptx_opcocde_latency_options (option_parser_t opp); + void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL = false ); + int gpgpu_opencl_ptx_sim_main_func( kernel_info_t *grid ); + void init_inst_classification_stat(); + kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, + gpgpu_ptx_sim_arg_list_t args, + struct dim3 gridDim, + struct dim3 blockDim, + gpgpu_t *gpu ); + void gpgpu_ptx_sim_register_global_variable(void *hostVar, const char *deviceName, size_t size ); + void gpgpu_ptx_sim_register_const_variable(void*, const char *deviceName, size_t size ); + void read_sim_environment_variables(); + void set_param_gpgpu_num_shaders(int num_shaders); + struct rec_pts find_reconvergence_points( function_info *finfo ); + address_type get_converge_point( address_type pc ); + void gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to, gpgpu_t *gpu ); + void ptx_print_insn( address_type pc, FILE *fp ); + std::string ptx_get_insn_str( address_type pc ); + template<int activate_level> bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc); }; #endif diff --git a/src/cuda-sim/cuda_device_printf.cc b/src/cuda-sim/cuda_device_printf.cc index 5088a88..9ac0727 100644 --- a/src/cuda-sim/cuda_device_printf.cc +++ b/src/cuda-sim/cuda_device_printf.cc @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -30,90 +28,87 @@ #include "cuda_device_printf.h" #include "ptx_ir.h" -void decode_space(memory_space_t &space, ptx_thread_info *thread, - const operand_info &op, memory_space *&mem, addr_t &addr); +void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand_info &op, memory_space *&mem, addr_t &addr); -void my_cuda_printf(const char *fmtstr, const char *arg_list) { - FILE *fp = stdout; - unsigned i = 0, j = 0; - unsigned arg_offset = 0; - char buf[64]; - bool in_fmt = false; - while (fmtstr[i]) { - char c = fmtstr[i++]; - if (!in_fmt) { - if (c != '%') { - fprintf(fp, "%c", c); +void my_cuda_printf(const char *fmtstr,const char *arg_list) +{ + FILE *fp = stdout; + unsigned i=0,j=0; + unsigned arg_offset=0; + char buf[64]; + bool in_fmt=false; + while( fmtstr[i] ) { + char c = fmtstr[i++]; + if( !in_fmt ) { + if( c != '%' ) { + fprintf(fp,"%c",c); + } else { + in_fmt=true; + buf[0] = c; + j=1; + } } else { - in_fmt = true; - buf[0] = c; - j = 1; + if(!( c == 'u' || c == 'f' || c == 'd' )) { + printf("GPGPU-Sim PTX: ERROR ** printf parsing support is limited to %%u, %%f, %%d at present"); + abort(); + } + buf[j] = c; + buf[j+1] = 0; + void* ptr = (void*)&arg_list[arg_offset]; + //unsigned long long value = ((unsigned long long*)arg_list)[arg_offset]; + if( c == 'u' || c == 'd' ) { + fprintf(fp,buf,*((unsigned long long*)ptr)); + } else if( c == 'f' ) { + double tmp = *((double*)ptr); + fprintf(fp,buf,tmp); + } + arg_offset++; + in_fmt=false; } - } else { - if (!(c == 'u' || c == 'f' || c == 'd')) { - printf( - "GPGPU-Sim PTX: ERROR ** printf parsing support is limited to %%u, " - "%%f, %%d at present"); - abort(); - } - buf[j] = c; - buf[j + 1] = 0; - void *ptr = (void *)&arg_list[arg_offset]; - // unsigned long long value = ((unsigned long long*)arg_list)[arg_offset]; - if (c == 'u' || c == 'd') { - fprintf(fp, buf, *((unsigned long long *)ptr)); - } else if (c == 'f') { - double tmp = *((double *)ptr); - fprintf(fp, buf, tmp); - } - arg_offset++; - in_fmt = false; - } - } + } } -void gpgpusim_cuda_vprintf(const ptx_instruction *pI, ptx_thread_info *thread, - const function_info *target_func) { - char *fmtstr = NULL; - char *arg_list = NULL; - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - assert(n_args == 2); - for (unsigned arg = 0; arg < n_args; arg++) { - const operand_info &actual_param_op = - pI->operand_lookup(n_return + 1 + arg); - const symbol *formal_param = target_func->get_arg(arg); - unsigned size = formal_param->get_size_in_bytes(); - assert(formal_param->is_param_local()); - assert(actual_param_op.is_param_local()); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - unsigned long long buffer[1024]; - assert(size < 1024 * sizeof(unsigned long long)); - thread->m_local_mem->read(from_addr, size, buffer); - addr_t addr = - (addr_t)buffer[0]; // should be pointer to generic memory location - memory_space *mem = NULL; - memory_space_t space = generic_space; - decode_space(space, thread, actual_param_op, mem, - addr); // figure out which space - if (arg == 0) { - unsigned len = 0; - char b = 0; - do { // figure out length - mem->read(addr + len, 1, &b); - len++; - } while (b); - fmtstr = (char *)malloc(len + 64); - for (int i = 0; i < len; i++) mem->read(addr + i, 1, fmtstr + i); - // mem->read(addr,len,fmtstr); - } else { - unsigned len = thread->get_finfo()->local_mem_framesize(); - arg_list = (char *)malloc(len + 64); - for (int i = 0; i < len; i++) mem->read(addr + i, 1, arg_list + i); - // mem->read(addr,len,arg_list); - } - } - my_cuda_printf(fmtstr, arg_list); - free(fmtstr); - free(arg_list); +void gpgpusim_cuda_vprintf(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func ) +{ + char *fmtstr = NULL; + char *arg_list = NULL; + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + assert( n_args == 2 ); + for( unsigned arg=0; arg < n_args; arg ++ ) { + const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); + const symbol *formal_param = target_func->get_arg(arg); + unsigned size=formal_param->get_size_in_bytes(); + assert( formal_param->is_param_local() ); + assert( actual_param_op.is_param_local() ); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + unsigned long long buffer[1024]; + assert(size<1024*sizeof(unsigned long long)); + thread->m_local_mem->read(from_addr,size,buffer); + addr_t addr = (addr_t)buffer[0]; // should be pointer to generic memory location + memory_space *mem=NULL; + memory_space_t space = generic_space; + decode_space(space,thread,actual_param_op,mem,addr); // figure out which space + if( arg == 0 ) { + unsigned len = 0; + char b = 0; + do { // figure out length + mem->read(addr+len,1,&b); + len++; + } while(b); + fmtstr = (char*)malloc(len+64); + for( int i=0; i < len; i++ ) + mem->read(addr+i,1,fmtstr+i); + //mem->read(addr,len,fmtstr); + } else { + unsigned len = thread->get_finfo()->local_mem_framesize(); + arg_list = (char*)malloc(len+64); + for( int i=0; i < len; i++ ) + mem->read(addr+i,1,arg_list+i); + //mem->read(addr,len,arg_list); + } + } + my_cuda_printf(fmtstr,arg_list); + free(fmtstr); + free(arg_list); } diff --git a/src/cuda-sim/cuda_device_printf.h b/src/cuda-sim/cuda_device_printf.h index 20744bb..4e9baaa 100644 --- a/src/cuda-sim/cuda_device_printf.h +++ b/src/cuda-sim/cuda_device_printf.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -30,8 +28,6 @@ #ifndef CUDA_DEVICE_PRINTF_INCLUDED #define CUDA_DEVICE_PRINTF_INCLUDED -void gpgpusim_cuda_vprintf(const class ptx_instruction* pI, - class ptx_thread_info* thread, - const class function_info* target_func); +void gpgpusim_cuda_vprintf(const class ptx_instruction * pI, class ptx_thread_info * thread, const class function_info * target_func ); #endif diff --git a/src/cuda-sim/cuda_device_runtime.cc b/src/cuda-sim/cuda_device_runtime.cc index 257dd50..4baced5 100644 --- a/src/cuda-sim/cuda_device_runtime.cc +++ b/src/cuda-sim/cuda_device_runtime.cc @@ -1,329 +1,297 @@ -// Jin: cuda_device_runtime.cc -// Defines CUDA device runtime APIs for CDP support +//Jin: cuda_device_runtime.cc +//Defines CUDA device runtime APIs for CDP support + #include <iostream> #include <map> + + #if (CUDART_VERSION >= 5000) #define __CUDA_RUNTIME_API_H__ #include <builtin_types.h> #include <driver_types.h> -#include "../../libcuda/gpgpu_context.h" #include "../gpgpu-sim/gpu-sim.h" -#include "../gpgpusim_entrypoint.h" -#include "../stream_manager.h" #include "cuda-sim.h" -#include "cuda_device_runtime.h" #include "ptx_ir.h" +#include "../stream_manager.h" +#include "../gpgpusim_entrypoint.h" +#include "cuda_device_runtime.h" +#include "../../libcuda/gpgpu_context.h" -#define DEV_RUNTIME_REPORT(a) \ - if (g_debug_execution) { \ - std::cout << __FILE__ << ", " << __LINE__ << ": " << a << "\n"; \ - std::cout.flush(); \ - } - -// Handling device runtime api: -// void * cudaGetParameterBufferV2(void *func, dim3 gridDimension, dim3 -// blockDimension, unsigned int sharedMemSize) -void cuda_device_runtime::gpgpusim_cuda_getParameterBufferV2( - const ptx_instruction *pI, ptx_thread_info *thread, - const function_info *target_func) { - DEV_RUNTIME_REPORT("Calling cudaGetParameterBufferV2"); +#define DEV_RUNTIME_REPORT(a) \ + if( g_debug_execution ) { \ + std::cout << __FILE__ << ", " << __LINE__ << ": " << a << "\n"; \ + std::cout.flush(); \ + } - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert(n_args == 4); - function_info *child_kernel_entry; - struct dim3 grid_dim, block_dim; - unsigned int shared_mem; - for (unsigned arg = 0; arg < n_args; arg++) { - const operand_info &actual_param_op = - pI->operand_lookup(n_return + 1 + arg); // param# - const symbol *formal_param = - target_func->get_arg(arg); // cudaGetParameterBufferV2_param_# - unsigned size = formal_param->get_size_in_bytes(); - assert(formal_param->is_param_local()); - assert(actual_param_op.is_param_local()); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); +//Handling device runtime api: +//void * cudaGetParameterBufferV2(void *func, dim3 gridDimension, dim3 blockDimension, unsigned int sharedMemSize) +void cuda_device_runtime::gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) +{ + DEV_RUNTIME_REPORT("Calling cudaGetParameterBufferV2"); + + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert( n_args == 4 ); - if (arg == 0) { // function_info* for the child kernel - unsigned long long buf; - assert(size == sizeof(function_info *)); - thread->m_local_mem->read(from_addr, size, &buf); - child_kernel_entry = (function_info *)buf; - assert(child_kernel_entry); - DEV_RUNTIME_REPORT("child kernel name " - << child_kernel_entry->get_name()); - } else if (arg == 1) { // dim3 grid_dim for the child kernel - assert(size == sizeof(struct dim3)); - thread->m_local_mem->read(from_addr, size, &grid_dim); - DEV_RUNTIME_REPORT("grid (" << grid_dim.x << ", " << grid_dim.y << ", " - << grid_dim.z << ")"); - } else if (arg == 2) { // dim3 block_dim for the child kernel - assert(size == sizeof(struct dim3)); - thread->m_local_mem->read(from_addr, size, &block_dim); - DEV_RUNTIME_REPORT("block (" << block_dim.x << ", " << block_dim.y << ", " - << block_dim.z << ")"); - } else if (arg == 3) { // unsigned int shared_mem - assert(size == sizeof(unsigned int)); - thread->m_local_mem->read(from_addr, size, &shared_mem); - DEV_RUNTIME_REPORT("shared memory " << shared_mem); - } - } + function_info * child_kernel_entry; + struct dim3 grid_dim, block_dim; + unsigned int shared_mem; - // get total child kernel argument size and malloc buffer in global memory - unsigned child_kernel_arg_size = child_kernel_entry->get_args_aligned_size(); - void *param_buffer = thread->get_gpu()->gpu_malloc(child_kernel_arg_size); - g_total_param_size += ((child_kernel_arg_size + 255) / 256 * 256); - DEV_RUNTIME_REPORT("child kernel arg size total " - << child_kernel_arg_size - << ", parameter buffer allocated at " << param_buffer); - if (g_total_param_size > g_max_total_param_size) - g_max_total_param_size = g_total_param_size; + for( unsigned arg=0; arg < n_args; arg ++ ) { + const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# + const symbol *formal_param = target_func->get_arg(arg); //cudaGetParameterBufferV2_param_# + unsigned size=formal_param->get_size_in_bytes(); + assert( formal_param->is_param_local() ); + assert( actual_param_op.is_param_local() ); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); - // store param buffer address and launch config - device_launch_config_t device_launch_config(grid_dim, block_dim, shared_mem, - child_kernel_entry); - assert(g_cuda_device_launch_param_map.find(param_buffer) == - g_cuda_device_launch_param_map.end()); - g_cuda_device_launch_param_map[param_buffer] = device_launch_config; + if(arg == 0) {//function_info* for the child kernel + unsigned long long buf; + assert(size == sizeof(function_info *)); + thread->m_local_mem->read(from_addr, size, &buf); + child_kernel_entry = (function_info *)buf; + assert(child_kernel_entry); + DEV_RUNTIME_REPORT("child kernel name " << child_kernel_entry->get_name()); + } + else if(arg == 1) { //dim3 grid_dim for the child kernel + assert(size == sizeof(struct dim3)); + thread->m_local_mem->read(from_addr, size, & grid_dim); + DEV_RUNTIME_REPORT("grid (" << grid_dim.x << ", " << grid_dim.y << ", " << grid_dim.z << ")"); + } + else if(arg == 2) { //dim3 block_dim for the child kernel + assert(size == sizeof(struct dim3)); + thread->m_local_mem->read(from_addr, size, & block_dim); + DEV_RUNTIME_REPORT("block (" << block_dim.x << ", " << block_dim.y << ", " << block_dim.z << ")"); + } + else if(arg == 3) { //unsigned int shared_mem + assert(size == sizeof(unsigned int)); + thread->m_local_mem->read(from_addr, size, & shared_mem); + DEV_RUNTIME_REPORT("shared memory " << shared_mem); + } + } - // copy the buffer address to retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 - const symbol *formal_return = target_func->get_return_var(); // void * - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaGetParameterBufferV2 return value has size of " - << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && - return_size == sizeof(void *)); - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, ¶m_buffer, NULL, - NULL); -} + //get total child kernel argument size and malloc buffer in global memory + unsigned child_kernel_arg_size = child_kernel_entry->get_args_aligned_size(); + void * param_buffer = thread->get_gpu()->gpu_malloc(child_kernel_arg_size); + g_total_param_size += ((child_kernel_arg_size + 255) / 256 * 256); + DEV_RUNTIME_REPORT("child kernel arg size total " << child_kernel_arg_size << ", parameter buffer allocated at " << param_buffer); + if(g_total_param_size > g_max_total_param_size) + g_max_total_param_size = g_total_param_size; -// Handling device runtime api: -// cudaError_t cudaLaunchDeviceV2(void *parameterBuffer, cudaStream_t stream) -void cuda_device_runtime::gpgpusim_cuda_launchDeviceV2( - const ptx_instruction *pI, ptx_thread_info *thread, - const function_info *target_func) { - DEV_RUNTIME_REPORT("Calling cudaLaunchDeviceV2"); + //store param buffer address and launch config + device_launch_config_t device_launch_config(grid_dim, block_dim, shared_mem, child_kernel_entry); + assert(g_cuda_device_launch_param_map.find(param_buffer) == g_cuda_device_launch_param_map.end()); + g_cuda_device_launch_param_map[param_buffer] = device_launch_config; - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert(n_args == 2); + //copy the buffer address to retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 + const symbol *formal_return = target_func->get_return_var(); //void * + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaGetParameterBufferV2 return value has size of " << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && return_size == sizeof(void *)); + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, ¶m_buffer, NULL, NULL); - kernel_info_t *device_grid = NULL; - function_info *device_kernel_entry = NULL; - void *parameter_buffer; - struct CUstream_st *child_stream; - device_launch_config_t config; - device_launch_operation_t device_launch_op; +} - for (unsigned arg = 0; arg < n_args; arg++) { - const operand_info &actual_param_op = - pI->operand_lookup(n_return + 1 + arg); // param# - const symbol *formal_param = - target_func->get_arg(arg); // cudaLaunchDeviceV2_param_# - unsigned size = formal_param->get_size_in_bytes(); - assert(formal_param->is_param_local()); - assert(actual_param_op.is_param_local()); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); +//Handling device runtime api: +//cudaError_t cudaLaunchDeviceV2(void *parameterBuffer, cudaStream_t stream) +void cuda_device_runtime::gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) { + DEV_RUNTIME_REPORT("Calling cudaLaunchDeviceV2"); - if (arg == 0) { // paramter buffer for child kernel (in global memory) - // get parameter_buffer from the cudaLaunchDeviceV2_param0 - assert(size == sizeof(void *)); - thread->m_local_mem->read(from_addr, size, ¶meter_buffer); - assert((size_t)parameter_buffer >= GLOBAL_HEAP_START); - DEV_RUNTIME_REPORT("Parameter buffer locating at global memory " - << parameter_buffer); + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert( n_args == 2 ); - // get child grid info through parameter_buffer address - assert(g_cuda_device_launch_param_map.find(parameter_buffer) != - g_cuda_device_launch_param_map.end()); - config = g_cuda_device_launch_param_map[parameter_buffer]; - // device_grid = op.grid; - device_kernel_entry = config.entry; - DEV_RUNTIME_REPORT("find device kernel " - << device_kernel_entry->get_name()); + kernel_info_t * device_grid = NULL; + function_info * device_kernel_entry = NULL; + void * parameter_buffer; + struct CUstream_st * child_stream; + device_launch_config_t config; + device_launch_operation_t device_launch_op; - // PDOM analysis is done for Parent kernel but not for child kernel. - if (device_kernel_entry->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", - device_kernel_entry->get_name().c_str()); - } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", - device_kernel_entry->get_name().c_str()); - /* - * Some of the instructions like printf() gives the gpgpusim the wrong - * impression that it is a function call. - * As printf() doesnt have a body like functions do, doing pdom analysis - * for printf() causes a crash. - */ - if (device_kernel_entry->get_function_size() > 0) - device_kernel_entry->do_pdom(); - device_kernel_entry->set_pdom(); - } + for( unsigned arg=0; arg < n_args; arg ++ ) { + const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# + const symbol *formal_param = target_func->get_arg(arg); //cudaLaunchDeviceV2_param_# + unsigned size=formal_param->get_size_in_bytes(); + assert( formal_param->is_param_local() ); + assert( actual_param_op.is_param_local() ); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); - // copy data in parameter_buffer to device kernel param memory - unsigned device_kernel_arg_size = - device_kernel_entry->get_args_aligned_size(); - DEV_RUNTIME_REPORT("device_kernel_arg_size " << device_kernel_arg_size); - memory_space *device_kernel_param_mem; + if(arg == 0) {//paramter buffer for child kernel (in global memory) + //get parameter_buffer from the cudaLaunchDeviceV2_param0 + assert(size == sizeof(void *)); + thread->m_local_mem->read(from_addr, size, ¶meter_buffer); + assert((size_t)parameter_buffer >= GLOBAL_HEAP_START); + DEV_RUNTIME_REPORT("Parameter buffer locating at global memory " << parameter_buffer); - // create child kernel_info_t and index it with parameter_buffer address - gpgpu_t *gpu = thread->get_gpu(); - device_grid = new kernel_info_t( - config.grid_dim, config.block_dim, device_kernel_entry, - gpu->getNameArrayMapping(), gpu->getNameInfoMapping()); - device_grid->launch_cycle = gpu->gpu_sim_cycle + gpu->gpu_tot_sim_cycle; - kernel_info_t &parent_grid = thread->get_kernel(); - DEV_RUNTIME_REPORT( - "child kernel launched by " - << parent_grid.name() << ", cta (" << thread->get_ctaid().x << ", " - << thread->get_ctaid().y << ", " << thread->get_ctaid().z - << "), thread (" << thread->get_tid().x << ", " << thread->get_tid().y - << ", " << thread->get_tid().z << ")"); - device_grid->set_parent(&parent_grid, thread->get_ctaid(), - thread->get_tid()); - device_launch_op = device_launch_operation_t(device_grid, NULL); - device_kernel_param_mem = device_grid->get_param_memory(); // kernel - // param - size_t param_start_address = 0; - // copy in word - for (unsigned n = 0; n < device_kernel_arg_size; n += 4) { - unsigned int oneword; - thread->get_gpu()->get_global_memory()->read( - (size_t)parameter_buffer + n, 4, &oneword); - device_kernel_param_mem->write(param_start_address + n, 4, &oneword, - NULL, NULL); - } - } else if (arg == 1) { // cudaStream for the child kernel + //get child grid info through parameter_buffer address + assert(g_cuda_device_launch_param_map.find(parameter_buffer) != g_cuda_device_launch_param_map.end()); + config = g_cuda_device_launch_param_map[parameter_buffer]; + //device_grid = op.grid; + device_kernel_entry = config.entry; + DEV_RUNTIME_REPORT("find device kernel " << device_kernel_entry->get_name()); + + //PDOM analysis is done for Parent kernel but not for child kernel. + if (device_kernel_entry->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", device_kernel_entry->get_name().c_str() ); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", device_kernel_entry->get_name().c_str() ); + /* + * Some of the instructions like printf() gives the gpgpusim the wrong impression that it is a function call. + * As printf() doesnt have a body like functions do, doing pdom analysis for printf() causes a crash. + */ + if (device_kernel_entry->get_function_size() >0) + device_kernel_entry->do_pdom(); + device_kernel_entry->set_pdom(); + } - assert(size == sizeof(cudaStream_t)); - thread->m_local_mem->read(from_addr, size, &child_stream); + //copy data in parameter_buffer to device kernel param memory + unsigned device_kernel_arg_size = device_kernel_entry->get_args_aligned_size(); + DEV_RUNTIME_REPORT("device_kernel_arg_size " << device_kernel_arg_size); + memory_space *device_kernel_param_mem; - kernel_info_t &parent_kernel = thread->get_kernel(); - if (child_stream == 0) { // default stream on device for current CTA - child_stream = - parent_kernel.get_default_stream_cta(thread->get_ctaid()); - DEV_RUNTIME_REPORT("launching child kernel " - << device_grid->get_uid() - << " to default stream of the cta " - << child_stream->get_uid() << ": " << child_stream); - } else { - assert(parent_kernel.cta_has_stream(thread->get_ctaid(), child_stream)); - DEV_RUNTIME_REPORT("launching child kernel " - << device_grid->get_uid() << " to stream " - << child_stream->get_uid() << ": " << child_stream); - } + //create child kernel_info_t and index it with parameter_buffer address + gpgpu_t* gpu=thread->get_gpu(); + device_grid = new kernel_info_t(config.grid_dim, config.block_dim, device_kernel_entry, gpu->getNameArrayMapping(), gpu->getNameInfoMapping()); + device_grid->launch_cycle = gpu->gpu_sim_cycle + gpu->gpu_tot_sim_cycle; + kernel_info_t & parent_grid = thread->get_kernel(); + DEV_RUNTIME_REPORT("child kernel launched by " << parent_grid.name() << ", cta (" << + thread->get_ctaid().x << ", " << thread->get_ctaid().y << ", " << thread->get_ctaid().z << + "), thread (" << thread->get_tid().x << ", " << thread->get_tid().y << ", " << thread->get_tid().z << + ")"); + device_grid->set_parent(&parent_grid, thread->get_ctaid(), thread->get_tid()); + device_launch_op = device_launch_operation_t(device_grid, NULL); + device_kernel_param_mem = device_grid->get_param_memory(); //kernel param + size_t param_start_address = 0; + //copy in word + for(unsigned n = 0; n < device_kernel_arg_size; n += 4) { + unsigned int oneword; + thread->get_gpu()->get_global_memory()->read((size_t)parameter_buffer + n, 4, &oneword); + device_kernel_param_mem->write(param_start_address + n, 4, &oneword, NULL, NULL); + } + } + else if(arg == 1) { //cudaStream for the child kernel - device_launch_op.stream = child_stream; + assert(size == sizeof(cudaStream_t)); + thread->m_local_mem->read(from_addr, size, &child_stream); + + kernel_info_t & parent_kernel = thread->get_kernel(); + if(child_stream == 0) { //default stream on device for current CTA + child_stream = parent_kernel.get_default_stream_cta(thread->get_ctaid()); + DEV_RUNTIME_REPORT("launching child kernel " << device_grid->get_uid() << + " to default stream of the cta " << child_stream->get_uid() << ": " << child_stream); + } + else { + assert(parent_kernel.cta_has_stream(thread->get_ctaid(), child_stream)); + DEV_RUNTIME_REPORT("launching child kernel " << device_grid->get_uid() << + " to stream " << child_stream->get_uid() << ": " << child_stream); + } + + device_launch_op.stream = child_stream; + } + } - } - // launch child kernel - g_cuda_device_launch_op.push_back(device_launch_op); - g_cuda_device_launch_param_map.erase(parameter_buffer); + + //launch child kernel + g_cuda_device_launch_op.push_back(device_launch_op); + g_cuda_device_launch_param_map.erase(parameter_buffer); + + //set retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 + const symbol *formal_return = target_func->get_return_var(); //cudaError_t + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaLaunchDeviceV2 return value has size of " << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size + && return_size == sizeof(cudaError_t)); + cudaError_t error = cudaSuccess; + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); - // set retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 - const symbol *formal_return = target_func->get_return_var(); // cudaError_t - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaLaunchDeviceV2 return value has size of " - << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && - return_size == sizeof(cudaError_t)); - cudaError_t error = cudaSuccess; - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); } -// Handling device runtime api: -// cudaError_t cudaStreamCreateWithFlags ( cudaStream_t* pStream, unsigned int -// flags) -// flags can only be cudaStreamNonBlocking -void cuda_device_runtime::gpgpusim_cuda_streamCreateWithFlags( - const ptx_instruction *pI, ptx_thread_info *thread, - const function_info *target_func) { - DEV_RUNTIME_REPORT("Calling cudaStreamCreateWithFlags"); - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert(n_args == 2); +//Handling device runtime api: +//cudaError_t cudaStreamCreateWithFlags ( cudaStream_t* pStream, unsigned int flags) +//flags can only be cudaStreamNonBlocking +void cuda_device_runtime::gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) { + DEV_RUNTIME_REPORT("Calling cudaStreamCreateWithFlags"); - size_t generic_pStream_addr; - addr_t pStream_addr; - unsigned int flags; - for (unsigned arg = 0; arg < n_args; arg++) { - const operand_info &actual_param_op = - pI->operand_lookup(n_return + 1 + arg); // param# - const symbol *formal_param = - target_func->get_arg(arg); // cudaStreamCreateWithFlags_param_# - unsigned size = formal_param->get_size_in_bytes(); - assert(formal_param->is_param_local()); - assert(actual_param_op.is_param_local()); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert( n_args == 2 ); - if (arg == 0) { // cudaStream_t * pStream, address of cudaStream_t - assert(size == sizeof(cudaStream_t *)); - thread->m_local_mem->read(from_addr, size, &generic_pStream_addr); + size_t generic_pStream_addr; + addr_t pStream_addr; + unsigned int flags; + for( unsigned arg=0; arg < n_args; arg ++ ) { + const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# + const symbol *formal_param = target_func->get_arg(arg); //cudaStreamCreateWithFlags_param_# + unsigned size=formal_param->get_size_in_bytes(); + assert( formal_param->is_param_local() ); + assert( actual_param_op.is_param_local() ); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); - // pStream should be non-zero address in local memory - pStream_addr = generic_to_local( - thread->get_hw_sid(), thread->get_hw_tid(), generic_pStream_addr); + if(arg == 0) {//cudaStream_t * pStream, address of cudaStream_t + assert(size == sizeof(cudaStream_t *)); + thread->m_local_mem->read(from_addr, size, &generic_pStream_addr); + + //pStream should be non-zero address in local memory + pStream_addr = generic_to_local(thread->get_hw_sid(), thread->get_hw_tid(), generic_pStream_addr); - DEV_RUNTIME_REPORT("pStream locating at local memory " << pStream_addr); - } else if (arg == - 1) { // unsigned int flags, should be cudaStreamNonBlocking - assert(size == sizeof(unsigned int)); - thread->m_local_mem->read(from_addr, size, &flags); - assert(flags == cudaStreamNonBlocking); + DEV_RUNTIME_REPORT("pStream locating at local memory " << pStream_addr); + } + else if(arg == 1) { //unsigned int flags, should be cudaStreamNonBlocking + assert(size == sizeof(unsigned int)); + thread->m_local_mem->read(from_addr, size, &flags); + assert(flags == cudaStreamNonBlocking); + } } - } - // create stream and write back to param0 - CUstream_st *stream = - thread->get_kernel().create_stream_cta(thread->get_ctaid()); - DEV_RUNTIME_REPORT("Create stream " << stream->get_uid() << ": " << stream); - thread->m_local_mem->write(pStream_addr, sizeof(cudaStream_t), &stream, NULL, - NULL); + //create stream and write back to param0 + CUstream_st * stream = thread->get_kernel().create_stream_cta(thread->get_ctaid()); + DEV_RUNTIME_REPORT("Create stream " << stream->get_uid() << ": " << stream); + thread->m_local_mem->write(pStream_addr, sizeof(cudaStream_t), &stream, NULL, NULL); + + //set retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 + const symbol *formal_return = target_func->get_return_var(); //cudaError_t + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaStreamCreateWithFlags return value has size of " << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size + && return_size == sizeof(cudaError_t)); + cudaError_t error = cudaSuccess; + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); - // set retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 - const symbol *formal_return = target_func->get_return_var(); // cudaError_t - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaStreamCreateWithFlags return value has size of " - << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && - return_size == sizeof(cudaError_t)); - cudaError_t error = cudaSuccess; - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); } + void cuda_device_runtime::launch_one_device_kernel() { - if (!g_cuda_device_launch_op.empty()) { - device_launch_operation_t &op = g_cuda_device_launch_op.front(); + if(!g_cuda_device_launch_op.empty()) { + device_launch_operation_t &op = g_cuda_device_launch_op.front(); - stream_operation stream_op = stream_operation( - op.grid, gpgpu_ctx->func_sim->g_ptx_sim_mode, op.stream); - gpgpu_ctx->the_gpgpusim->g_stream_manager->push(stream_op); - g_cuda_device_launch_op.pop_front(); - } + stream_operation stream_op = stream_operation(op.grid, gpgpu_ctx->func_sim->g_ptx_sim_mode, op.stream); + gpgpu_ctx->the_gpgpusim->g_stream_manager->push(stream_op); + g_cuda_device_launch_op.pop_front(); + } } void cuda_device_runtime::launch_all_device_kernels() { - while (!g_cuda_device_launch_op.empty()) { - launch_one_device_kernel(); - } + while(!g_cuda_device_launch_op.empty()) { + launch_one_device_kernel(); + } } #endif diff --git a/src/cuda-sim/cuda_device_runtime.h b/src/cuda-sim/cuda_device_runtime.h index 1d661b2..7f7a0ca 100644 --- a/src/cuda-sim/cuda_device_runtime.h +++ b/src/cuda-sim/cuda_device_runtime.h @@ -1,66 +1,67 @@ #ifndef __cuda_device_runtime_h__ #define __cuda_device_runtime_h__ -// Jin: cuda_device_runtime.h -// Defines CUDA device runtime APIs for CDP support +//Jin: cuda_device_runtime.h +//Defines CUDA device runtime APIs for CDP support class device_launch_config_t { - public: - device_launch_config_t() {} - device_launch_config_t(dim3 _grid_dim, dim3 _block_dim, - unsigned int _shared_mem, function_info* _entry) - : grid_dim(_grid_dim), - block_dim(_block_dim), - shared_mem(_shared_mem), - entry(_entry) {} +public: + device_launch_config_t() {} + + device_launch_config_t(dim3 _grid_dim, + dim3 _block_dim, + unsigned int _shared_mem, + function_info * _entry): + grid_dim(_grid_dim), + block_dim(_block_dim), + shared_mem(_shared_mem), + entry(_entry) {} + + dim3 grid_dim; + dim3 block_dim; + unsigned int shared_mem; + function_info * entry; - dim3 grid_dim; - dim3 block_dim; - unsigned int shared_mem; - function_info* entry; }; class device_launch_operation_t { - public: - device_launch_operation_t() {} - device_launch_operation_t(kernel_info_t* _grid, CUstream_st* _stream) - : grid(_grid), stream(_stream) {} - kernel_info_t* grid; // a new child grid +public: + device_launch_operation_t() {} + device_launch_operation_t(kernel_info_t *_grid, + CUstream_st * _stream) : + grid(_grid), stream(_stream) {} + + kernel_info_t * grid; //a new child grid + + CUstream_st * stream; - CUstream_st* stream; }; class gpgpu_context; class cuda_device_runtime { - public: - cuda_device_runtime(gpgpu_context* ctx) { - g_total_param_size = 0; - g_max_total_param_size = 0; - gpgpu_ctx = ctx; - } - unsigned long long g_total_param_size; - std::map<void*, device_launch_config_t> g_cuda_device_launch_param_map; - std::list<device_launch_operation_t> g_cuda_device_launch_op; - unsigned g_kernel_launch_latency; - unsigned long long g_max_total_param_size; - bool g_cdp_enabled; + public: + cuda_device_runtime( gpgpu_context* ctx ) { + g_total_param_size = 0; + g_max_total_param_size = 0; + gpgpu_ctx = ctx; + } + unsigned long long g_total_param_size; + std::map<void *, device_launch_config_t> g_cuda_device_launch_param_map; + std::list<device_launch_operation_t> g_cuda_device_launch_op; + unsigned g_kernel_launch_latency; + unsigned long long g_max_total_param_size; + bool g_cdp_enabled; - // backward pointer - class gpgpu_context* gpgpu_ctx; + // backward pointer + class gpgpu_context* gpgpu_ctx; #if (CUDART_VERSION >= 5000) #pragma once - void gpgpusim_cuda_launchDeviceV2(const ptx_instruction* pI, - ptx_thread_info* thread, - const function_info* target_func); - void gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction* pI, - ptx_thread_info* thread, - const function_info* target_func); - void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction* pI, - ptx_thread_info* thread, - const function_info* target_func); - void launch_all_device_kernels(); - void launch_one_device_kernel(); + void gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); + void gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); + void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); + void launch_all_device_kernels(); + void launch_one_device_kernel(); #endif }; diff --git a/src/cuda-sim/half.h b/src/cuda-sim/half.h index d33b03c..9f74bb7 100644 --- a/src/cuda-sim/half.h +++ b/src/cuda-sim/half.h @@ -2,25 +2,17 @@ //
// Copyright (c) 2012-2017 Christian Rau <[email protected]>
//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation
-// files (the "Software"), to deal in the Software without restriction,
-// including without limitation the rights to use, copy,
-// modify, merge, publish, distribute, sublicense, and/or sell copies of the
-// Software, and to permit persons to whom the
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy,
+// modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
-// The above copyright notice and this permission notice shall be included in
-// all copies or substantial portions of the Software.
+// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE
-// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-// DEALINGS IN THE SOFTWARE.
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
// Version 1.12.0
@@ -31,191 +23,180 @@ #define HALF_HALF_HPP
/// Combined gcc version number.
-#define HALF_GNUC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+#define HALF_GNUC_VERSION (__GNUC__*100+__GNUC_MINOR__)
-// check C++11 language features
-#if defined(__clang__) // clang
-#if __has_feature(cxx_static_assert) && \
- !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
-#define HALF_ENABLE_CPP11_STATIC_ASSERT 1
-#endif
-#if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
-#define HALF_ENABLE_CPP11_CONSTEXPR 1
-#endif
-#if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
-#define HALF_ENABLE_CPP11_NOEXCEPT 1
-#endif
-#if __has_feature(cxx_user_literals) && \
- !defined(HALF_ENABLE_CPP11_USER_LITERALS)
-#define HALF_ENABLE_CPP11_USER_LITERALS 1
-#endif
-#if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && \
- !defined(HALF_ENABLE_CPP11_LONG_LONG)
-#define HALF_ENABLE_CPP11_LONG_LONG 1
-#endif
-/*#elif defined(__INTEL_COMPILER)
- //Intel C++
- #if __INTEL_COMPILER >= 1100 &&
- !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) ????????
- #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
- #endif
- #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
- ????????
- #define HALF_ENABLE_CPP11_CONSTEXPR 1
- #endif
- #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
- ????????
- #define HALF_ENABLE_CPP11_NOEXCEPT 1
- #endif
- #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_LONG_LONG)
- ????????
- #define HALF_ENABLE_CPP11_LONG_LONG 1
- #endif*/
-#elif defined(__GNUC__) // gcc
-#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L
-#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
-#define HALF_ENABLE_CPP11_STATIC_ASSERT 1
-#endif
-#if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
-#define HALF_ENABLE_CPP11_CONSTEXPR 1
-#endif
-#if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
-#define HALF_ENABLE_CPP11_NOEXCEPT 1
-#endif
-#if HALF_GNUC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
-#define HALF_ENABLE_CPP11_USER_LITERALS 1
-#endif
-#if !defined(HALF_ENABLE_CPP11_LONG_LONG)
-#define HALF_ENABLE_CPP11_LONG_LONG 1
-#endif
-#endif
-#elif defined(_MSC_VER) // Visual C++
-#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
-#define HALF_ENABLE_CPP11_CONSTEXPR 1
-#endif
-#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
-#define HALF_ENABLE_CPP11_NOEXCEPT 1
-#endif
-#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
-#define HALF_ENABLE_CPP11_USER_LITERALS 1
-#endif
-#if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
-#define HALF_ENABLE_CPP11_STATIC_ASSERT 1
-#endif
-#if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG)
-#define HALF_ENABLE_CPP11_LONG_LONG 1
-#endif
-#define HALF_POP_WARNINGS 1
-#pragma warning(push)
-#pragma warning(disable : 4099 4127 4146) // struct vs class, constant in if,
- // negative unsigned
+//check C++11 language features
+#if defined(__clang__) //clang
+ #if __has_feature(cxx_static_assert) && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if __has_feature(cxx_user_literals) && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+ #define HALF_ENABLE_CPP11_USER_LITERALS 1
+ #endif
+ #if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && !defined(HALF_ENABLE_CPP11_LONG_LONG)
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif
+/*#elif defined(__INTEL_COMPILER) //Intel C++
+ #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) ????????
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) ????????
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) ????????
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_LONG_LONG) ????????
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif*/
+#elif defined(__GNUC__) //gcc
+ #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if HALF_GNUC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+ #define HALF_ENABLE_CPP11_USER_LITERALS 1
+ #endif
+ #if !defined(HALF_ENABLE_CPP11_LONG_LONG)
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif
+ #endif
+#elif defined(_MSC_VER) //Visual C++
+ #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+ #define HALF_ENABLE_CPP11_USER_LITERALS 1
+ #endif
+ #if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG)
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif
+ #define HALF_POP_WARNINGS 1
+ #pragma warning(push)
+ #pragma warning(disable : 4099 4127 4146) //struct vs class, constant in if, negative unsigned
#endif
-// check C++11 library features
+//check C++11 library features
#include <utility>
-#if defined(_LIBCPP_VERSION) // libc++
-#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103
-#ifndef HALF_ENABLE_CPP11_TYPE_TRAITS
-#define HALF_ENABLE_CPP11_TYPE_TRAITS 1
-#endif
-#ifndef HALF_ENABLE_CPP11_CSTDINT
-#define HALF_ENABLE_CPP11_CSTDINT 1
-#endif
-#ifndef HALF_ENABLE_CPP11_CMATH
-#define HALF_ENABLE_CPP11_CMATH 1
-#endif
-#ifndef HALF_ENABLE_CPP11_HASH
-#define HALF_ENABLE_CPP11_HASH 1
-#endif
-#endif
-#elif defined(__GLIBCXX__) // libstdc++
-#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103
-#ifdef __clang__
-#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS)
-#define HALF_ENABLE_CPP11_TYPE_TRAITS 1
-#endif
-#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT)
-#define HALF_ENABLE_CPP11_CSTDINT 1
-#endif
-#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH)
-#define HALF_ENABLE_CPP11_CMATH 1
-#endif
-#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH)
-#define HALF_ENABLE_CPP11_HASH 1
-#endif
-#else
-#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT)
-#define HALF_ENABLE_CPP11_CSTDINT 1
-#endif
-#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH)
-#define HALF_ENABLE_CPP11_CMATH 1
-#endif
-#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH)
-#define HALF_ENABLE_CPP11_HASH 1
-#endif
-#endif
-#endif
-#elif defined(_CPPLIB_VER) // Dinkumware/Visual C++
-#if _CPPLIB_VER >= 520
-#ifndef HALF_ENABLE_CPP11_TYPE_TRAITS
-#define HALF_ENABLE_CPP11_TYPE_TRAITS 1
-#endif
-#ifndef HALF_ENABLE_CPP11_CSTDINT
-#define HALF_ENABLE_CPP11_CSTDINT 1
-#endif
-#ifndef HALF_ENABLE_CPP11_HASH
-#define HALF_ENABLE_CPP11_HASH 1
-#endif
-#endif
-#if _CPPLIB_VER >= 610
-#ifndef HALF_ENABLE_CPP11_CMATH
-#define HALF_ENABLE_CPP11_CMATH 1
-#endif
-#endif
+#if defined(_LIBCPP_VERSION) //libc++
+ #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103
+ #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS
+ #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_CSTDINT
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_CMATH
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_HASH
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #endif
+#elif defined(__GLIBCXX__) //libstdc++
+ #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103
+ #ifdef __clang__
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS)
+ #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+ #endif
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT)
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH)
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH)
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #else
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT)
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH)
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH)
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #endif
+ #endif
+#elif defined(_CPPLIB_VER) //Dinkumware/Visual C++
+ #if _CPPLIB_VER >= 520
+ #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS
+ #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_CSTDINT
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_HASH
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #endif
+ #if _CPPLIB_VER >= 610
+ #ifndef HALF_ENABLE_CPP11_CMATH
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #endif
#endif
#undef HALF_GNUC_VERSION
-// support constexpr
+//support constexpr
#if HALF_ENABLE_CPP11_CONSTEXPR
-#define HALF_CONSTEXPR constexpr
-#define HALF_CONSTEXPR_CONST constexpr
+ #define HALF_CONSTEXPR constexpr
+ #define HALF_CONSTEXPR_CONST constexpr
#else
-#define HALF_CONSTEXPR
-#define HALF_CONSTEXPR_CONST const
+ #define HALF_CONSTEXPR
+ #define HALF_CONSTEXPR_CONST const
#endif
-// support noexcept
+//support noexcept
#if HALF_ENABLE_CPP11_NOEXCEPT
-#define HALF_NOEXCEPT noexcept
-#define HALF_NOTHROW noexcept
+ #define HALF_NOEXCEPT noexcept
+ #define HALF_NOTHROW noexcept
#else
-#define HALF_NOEXCEPT
-#define HALF_NOTHROW throw()
+ #define HALF_NOEXCEPT
+ #define HALF_NOTHROW throw()
#endif
#include <algorithm>
+#include <iostream>
+#include <limits>
#include <climits>
#include <cmath>
#include <cstring>
-#include <iostream>
-#include <limits>
#if HALF_ENABLE_CPP11_TYPE_TRAITS
-#include <type_traits>
+ #include <type_traits>
#endif
#if HALF_ENABLE_CPP11_CSTDINT
-#include <cstdint>
+ #include <cstdint>
#endif
#if HALF_ENABLE_CPP11_HASH
-#include <functional>
+ #include <functional>
#endif
+
/// Default rounding mode.
-/// This specifies the rounding mode used for all conversions between
-/// [half](\ref half_float::half)s and `float`s as well as
-/// for the half_cast() if not specifying a rounding mode explicitly. It can be
-/// redefined (before including half.hpp) to one
-/// of the standard rounding modes using their respective constants or the
-/// equivalent values of `std::float_round_style`:
+/// This specifies the rounding mode used for all conversions between [half](\ref half_float::half)s and `float`s as well as
+/// for the half_cast() if not specifying a rounding mode explicitly. It can be redefined (before including half.hpp) to one
+/// of the standard rounding modes using their respective constants or the equivalent values of `std::float_round_style`:
///
/// `std::float_round_style` | value | rounding
/// ---------------------------------|-------|-------------------------
@@ -225,354 +206,256 @@ /// `std::round_toward_infinity` | 2 | toward positive infinity
/// `std::round_toward_neg_infinity` | 3 | toward negative infinity
///
-/// By default this is set to `-1` (`std::round_indeterminate`), which uses
-/// truncation (round toward zero, but with overflows
-/// set to infinity) and is the fastest rounding mode possible. It can even be
-/// set to `std::numeric_limits<float>::round_style`
-/// to synchronize the rounding mode with that of the underlying
-/// single-precision implementation.
+/// By default this is set to `-1` (`std::round_indeterminate`), which uses truncation (round toward zero, but with overflows
+/// set to infinity) and is the fastest rounding mode possible. It can even be set to `std::numeric_limits<float>::round_style`
+/// to synchronize the rounding mode with that of the underlying single-precision implementation.
#ifndef HALF_ROUND_STYLE
-#define HALF_ROUND_STYLE -1 // = std::round_indeterminate
+ #define HALF_ROUND_STYLE -1 // = std::round_indeterminate
#endif
/// Tie-breaking behaviour for round to nearest.
-/// This specifies if ties in round to nearest should be resolved by rounding to
-/// the nearest even value. By default this is
-/// defined to `0` resulting in the faster but slightly more biased behaviour of
-/// rounding away from zero in half-way cases (and
-/// thus equal to the round() function), but can be redefined to `1` (before
-/// including half.hpp) if more IEEE-conformant
+/// This specifies if ties in round to nearest should be resolved by rounding to the nearest even value. By default this is
+/// defined to `0` resulting in the faster but slightly more biased behaviour of rounding away from zero in half-way cases (and
+/// thus equal to the round() function), but can be redefined to `1` (before including half.hpp) if more IEEE-conformant
/// behaviour is needed.
#ifndef HALF_ROUND_TIES_TO_EVEN
-#define HALF_ROUND_TIES_TO_EVEN 0 // ties away from zero
+ #define HALF_ROUND_TIES_TO_EVEN 0 // ties away from zero
#endif
/// Value signaling overflow.
-/// In correspondence with `HUGE_VAL[F|L]` from `<cmath>` this symbol expands to
-/// a positive value signaling the overflow of an
+/// In correspondence with `HUGE_VAL[F|L]` from `<cmath>` this symbol expands to a positive value signaling the overflow of an
/// operation, in particular it just evaluates to positive infinity.
-#define HUGE_VALH std::numeric_limits<half_float::half>::infinity()
+#define HUGE_VALH std::numeric_limits<half_float::half>::infinity()
/// Fast half-precision fma function.
-/// This symbol is only defined if the fma() function generally executes as fast
-/// as, or faster than, a separate
-/// half-precision multiplication followed by an addition. Due to the internal
-/// single-precision implementation of all
+/// This symbol is only defined if the fma() function generally executes as fast as, or faster than, a separate
+/// half-precision multiplication followed by an addition. Due to the internal single-precision implementation of all
/// arithmetic operations, this is in fact always the case.
-#define FP_FAST_FMAH 1
+#define FP_FAST_FMAH 1
#ifndef FP_ILOGB0
-#define FP_ILOGB0 INT_MIN
+ #define FP_ILOGB0 INT_MIN
#endif
#ifndef FP_ILOGBNAN
-#define FP_ILOGBNAN INT_MAX
+ #define FP_ILOGBNAN INT_MAX
#endif
#ifndef FP_SUBNORMAL
-#define FP_SUBNORMAL 0
+ #define FP_SUBNORMAL 0
#endif
#ifndef FP_ZERO
-#define FP_ZERO 1
+ #define FP_ZERO 1
#endif
#ifndef FP_NAN
-#define FP_NAN 2
+ #define FP_NAN 2
#endif
#ifndef FP_INFINITE
-#define FP_INFINITE 3
+ #define FP_INFINITE 3
#endif
#ifndef FP_NORMAL
-#define FP_NORMAL 4
+ #define FP_NORMAL 4
#endif
+
/// Main namespace for half precision functionality.
/// This namespace contains all the functionality provided by the library.
-namespace half_float {
-class half;
+namespace half_float
+{
+ class half;
#if HALF_ENABLE_CPP11_USER_LITERALS
-/// Library-defined half-precision literals.
-/// Import this namespace to enable half-precision floating point literals:
-/// ~~~~{.cpp}
-/// using namespace half_float::literal;
-/// half_float::half = 4.2_h;
-/// ~~~~
-namespace literal {
-half operator"" _h(long double);
-}
+ /// Library-defined half-precision literals.
+ /// Import this namespace to enable half-precision floating point literals:
+ /// ~~~~{.cpp}
+ /// using namespace half_float::literal;
+ /// half_float::half = 4.2_h;
+ /// ~~~~
+ namespace literal
+ {
+ half operator"" _h(long double);
+ }
#endif
-/// \internal
-/// \brief Implementation details.
-namespace detail {
-#if HALF_ENABLE_CPP11_TYPE_TRAITS
-/// Conditional type.
-template <bool B, typename T, typename F>
-struct conditional : std::conditional<B, T, F> {};
+ /// \internal
+ /// \brief Implementation details.
+ namespace detail
+ {
+ #if HALF_ENABLE_CPP11_TYPE_TRAITS
+ /// Conditional type.
+ template<bool B,typename T,typename F> struct conditional : std::conditional<B,T,F> {};
-/// Helper for tag dispatching.
-template <bool B>
-struct bool_type : std::integral_constant<bool, B> {};
-using std::true_type;
-using std::false_type;
+ /// Helper for tag dispatching.
+ template<bool B> struct bool_type : std::integral_constant<bool,B> {};
+ using std::true_type;
+ using std::false_type;
-/// Type traits for floating point types.
-template <typename T>
-struct is_float : std::is_floating_point<T> {};
-#else
-/// Conditional type.
-template <bool, typename T, typename>
-struct conditional {
- typedef T type;
-};
-template <typename T, typename F>
-struct conditional<false, T, F> {
- typedef F type;
-};
+ /// Type traits for floating point types.
+ template<typename T> struct is_float : std::is_floating_point<T> {};
+ #else
+ /// Conditional type.
+ template<bool,typename T,typename> struct conditional { typedef T type; };
+ template<typename T,typename F> struct conditional<false,T,F> { typedef F type; };
-/// Helper for tag dispatching.
-template <bool>
-struct bool_type {};
-typedef bool_type<true> true_type;
-typedef bool_type<false> false_type;
+ /// Helper for tag dispatching.
+ template<bool> struct bool_type {};
+ typedef bool_type<true> true_type;
+ typedef bool_type<false> false_type;
-/// Type traits for floating point types.
-template <typename>
-struct is_float : false_type {};
-template <typename T>
-struct is_float<const T> : is_float<T> {};
-template <typename T>
-struct is_float<volatile T> : is_float<T> {};
-template <typename T>
-struct is_float<const volatile T> : is_float<T> {};
-template <>
-struct is_float<float> : true_type {};
-template <>
-struct is_float<double> : true_type {};
-template <>
-struct is_float<long double> : true_type {};
-#endif
+ /// Type traits for floating point types.
+ template<typename> struct is_float : false_type {};
+ template<typename T> struct is_float<const T> : is_float<T> {};
+ template<typename T> struct is_float<volatile T> : is_float<T> {};
+ template<typename T> struct is_float<const volatile T> : is_float<T> {};
+ template<> struct is_float<float> : true_type {};
+ template<> struct is_float<double> : true_type {};
+ template<> struct is_float<long double> : true_type {};
+ #endif
-/// Type traits for floating point bits.
-template <typename T>
-struct bits {
- typedef unsigned char type;
-};
-template <typename T>
-struct bits<const T> : bits<T> {};
-template <typename T>
-struct bits<volatile T> : bits<T> {};
-template <typename T>
-struct bits<const volatile T> : bits<T> {};
+ /// Type traits for floating point bits.
+ template<typename T> struct bits { typedef unsigned char type; };
+ template<typename T> struct bits<const T> : bits<T> {};
+ template<typename T> struct bits<volatile T> : bits<T> {};
+ template<typename T> struct bits<const volatile T> : bits<T> {};
-#if HALF_ENABLE_CPP11_CSTDINT
-/// Unsigned integer of (at least) 16 bits width.
-typedef std::uint_least16_t uint16;
+ #if HALF_ENABLE_CPP11_CSTDINT
+ /// Unsigned integer of (at least) 16 bits width.
+ typedef std::uint_least16_t uint16;
-/// Unsigned integer of (at least) 32 bits width.
-template <>
-struct bits<float> {
- typedef std::uint_least32_t type;
-};
+ /// Unsigned integer of (at least) 32 bits width.
+ template<> struct bits<float> { typedef std::uint_least32_t type; };
-/// Unsigned integer of (at least) 64 bits width.
-template <>
-struct bits<double> {
- typedef std::uint_least64_t type;
-};
-#else
-/// Unsigned integer of (at least) 16 bits width.
-typedef unsigned short uint16;
+ /// Unsigned integer of (at least) 64 bits width.
+ template<> struct bits<double> { typedef std::uint_least64_t type; };
+ #else
+ /// Unsigned integer of (at least) 16 bits width.
+ typedef unsigned short uint16;
-/// Unsigned integer of (at least) 32 bits width.
-template <>
-struct bits<float>
- : conditional<std::numeric_limits<unsigned int>::digits >= 32, unsigned int,
- unsigned long> {};
+ /// Unsigned integer of (at least) 32 bits width.
+ template<> struct bits<float> : conditional<std::numeric_limits<unsigned int>::digits>=32,unsigned int,unsigned long> {};
-#if HALF_ENABLE_CPP11_LONG_LONG
-/// Unsigned integer of (at least) 64 bits width.
-template <>
-struct bits<double>
- : conditional<std::numeric_limits<unsigned long>::digits >= 64,
- unsigned long, unsigned long long> {};
-#else
-/// Unsigned integer of (at least) 64 bits width.
-template <>
-struct bits<double> {
- typedef unsigned long type;
-};
-#endif
-#endif
+ #if HALF_ENABLE_CPP11_LONG_LONG
+ /// Unsigned integer of (at least) 64 bits width.
+ template<> struct bits<double> : conditional<std::numeric_limits<unsigned long>::digits>=64,unsigned long,unsigned long long> {};
+ #else
+ /// Unsigned integer of (at least) 64 bits width.
+ template<> struct bits<double> { typedef unsigned long type; };
+ #endif
+ #endif
-/// Tag type for binary construction.
-struct binary_t {};
+ /// Tag type for binary construction.
+ struct binary_t {};
-/// Tag for binary construction.
-HALF_CONSTEXPR_CONST binary_t binary = binary_t();
+ /// Tag for binary construction.
+ HALF_CONSTEXPR_CONST binary_t binary = binary_t();
-/// Temporary half-precision expression.
-/// This class represents a half-precision expression which just stores a
-/// single-precision value internally.
-struct expr {
- /// Conversion constructor.
- /// \param f single-precision value to convert
- explicit HALF_CONSTEXPR expr(float f) HALF_NOEXCEPT : value_(f) {}
+ /// Temporary half-precision expression.
+ /// This class represents a half-precision expression which just stores a single-precision value internally.
+ struct expr
+ {
+ /// Conversion constructor.
+ /// \param f single-precision value to convert
+ explicit HALF_CONSTEXPR expr(float f) HALF_NOEXCEPT : value_(f) {}
- /// Conversion to single-precision.
- /// \return single precision value representing expression value
- HALF_CONSTEXPR operator float() const HALF_NOEXCEPT { return value_; }
+ /// Conversion to single-precision.
+ /// \return single precision value representing expression value
+ HALF_CONSTEXPR operator float() const HALF_NOEXCEPT { return value_; }
- private:
- /// Internal expression value stored in single-precision.
- float value_;
-};
+ private:
+ /// Internal expression value stored in single-precision.
+ float value_;
+ };
-/// SFINAE helper for generic half-precision functions.
-/// This class template has to be specialized for each valid combination of
-/// argument types to provide a corresponding
-/// `type` member equivalent to \a T.
-/// \tparam T type to return
-template <typename T, typename, typename = void, typename = void>
-struct enable {};
-template <typename T>
-struct enable<T, half, void, void> {
- typedef T type;
-};
-template <typename T>
-struct enable<T, expr, void, void> {
- typedef T type;
-};
-template <typename T>
-struct enable<T, half, half, void> {
- typedef T type;
-};
-template <typename T>
-struct enable<T, half, expr, void> {
- typedef T type;
-};
-template <typename T>
-struct enable<T, expr, half, void> {
- typedef T type;
-};
-template <typename T>
-struct enable<T, expr, expr, void> {
- typedef T type;
-};
-template <typename T>
-struct enable<T, half, half, half> {
- typedef T type;
-};
-template <typename T>
-struct enable<T, half, half, expr> {
- typedef T type;
-};
-template <typename T>
-struct enable<T, half, expr, half> {
- typedef T type;
-};
-template <typename T>
-struct enable<T, half, expr, expr> {
- typedef T type;
-};
-template <typename T>
-struct enable<T, expr, half, half> {
- typedef T type;
-};
-template <typename T>
-struct enable<T, expr, half, expr> {
- typedef T type;
-};
-template <typename T>
-struct enable<T, expr, expr, half> {
- typedef T type;
-};
-template <typename T>
-struct enable<T, expr, expr, expr> {
- typedef T type;
-};
+ /// SFINAE helper for generic half-precision functions.
+ /// This class template has to be specialized for each valid combination of argument types to provide a corresponding
+ /// `type` member equivalent to \a T.
+ /// \tparam T type to return
+ template<typename T,typename,typename=void,typename=void> struct enable {};
+ template<typename T> struct enable<T,half,void,void> { typedef T type; };
+ template<typename T> struct enable<T,expr,void,void> { typedef T type; };
+ template<typename T> struct enable<T,half,half,void> { typedef T type; };
+ template<typename T> struct enable<T,half,expr,void> { typedef T type; };
+ template<typename T> struct enable<T,expr,half,void> { typedef T type; };
+ template<typename T> struct enable<T,expr,expr,void> { typedef T type; };
+ template<typename T> struct enable<T,half,half,half> { typedef T type; };
+ template<typename T> struct enable<T,half,half,expr> { typedef T type; };
+ template<typename T> struct enable<T,half,expr,half> { typedef T type; };
+ template<typename T> struct enable<T,half,expr,expr> { typedef T type; };
+ template<typename T> struct enable<T,expr,half,half> { typedef T type; };
+ template<typename T> struct enable<T,expr,half,expr> { typedef T type; };
+ template<typename T> struct enable<T,expr,expr,half> { typedef T type; };
+ template<typename T> struct enable<T,expr,expr,expr> { typedef T type; };
-/// Return type for specialized generic 2-argument half-precision functions.
-/// This class template has to be specialized for each valid combination of
-/// argument types to provide a corresponding
-/// `type` member denoting the appropriate return type.
-/// \tparam T first argument type
-/// \tparam U first argument type
-template <typename T, typename U>
-struct result : enable<expr, T, U> {};
-template <>
-struct result<half, half> {
- typedef half type;
-};
+ /// Return type for specialized generic 2-argument half-precision functions.
+ /// This class template has to be specialized for each valid combination of argument types to provide a corresponding
+ /// `type` member denoting the appropriate return type.
+ /// \tparam T first argument type
+ /// \tparam U first argument type
+ template<typename T,typename U> struct result : enable<expr,T,U> {};
+ template<> struct result<half,half> { typedef half type; };
-/// \name Classification helpers
-/// \{
+ /// \name Classification helpers
+ /// \{
-/// Check for infinity.
-/// \tparam T argument type (builtin floating point type)
-/// \param arg value to query
-/// \retval true if infinity
-/// \retval false else
-template <typename T>
-bool builtin_isinf(T arg) {
-#if HALF_ENABLE_CPP11_CMATH
- return std::isinf(arg);
-#elif defined(_MSC_VER)
- return !::_finite(static_cast<double>(arg)) &&
- !::_isnan(static_cast<double>(arg));
-#else
- return arg == std::numeric_limits<T>::infinity() ||
- arg == -std::numeric_limits<T>::infinity();
-#endif
-}
+ /// Check for infinity.
+ /// \tparam T argument type (builtin floating point type)
+ /// \param arg value to query
+ /// \retval true if infinity
+ /// \retval false else
+ template<typename T> bool builtin_isinf(T arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return std::isinf(arg);
+ #elif defined(_MSC_VER)
+ return !::_finite(static_cast<double>(arg)) && !::_isnan(static_cast<double>(arg));
+ #else
+ return arg == std::numeric_limits<T>::infinity() || arg == -std::numeric_limits<T>::infinity();
+ #endif
+ }
-/// Check for NaN.
-/// \tparam T argument type (builtin floating point type)
-/// \param arg value to query
-/// \retval true if not a number
-/// \retval false else
-template <typename T>
-bool builtin_isnan(T arg) {
-#if HALF_ENABLE_CPP11_CMATH
- return std::isnan(arg);
-#elif defined(_MSC_VER)
- return ::_isnan(static_cast<double>(arg)) != 0;
-#else
- return arg != arg;
-#endif
-}
+ /// Check for NaN.
+ /// \tparam T argument type (builtin floating point type)
+ /// \param arg value to query
+ /// \retval true if not a number
+ /// \retval false else
+ template<typename T> bool builtin_isnan(T arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return std::isnan(arg);
+ #elif defined(_MSC_VER)
+ return ::_isnan(static_cast<double>(arg)) != 0;
+ #else
+ return arg != arg;
+ #endif
+ }
-/// Check sign.
-/// \tparam T argument type (builtin floating point type)
-/// \param arg value to query
-/// \retval true if signbit set
-/// \retval false else
-template <typename T>
-bool builtin_signbit(T arg) {
-#if HALF_ENABLE_CPP11_CMATH
- return std::signbit(arg);
-#else
- return arg < T() || (arg == T() && T(1) / arg < T());
-#endif
-}
+ /// Check sign.
+ /// \tparam T argument type (builtin floating point type)
+ /// \param arg value to query
+ /// \retval true if signbit set
+ /// \retval false else
+ template<typename T> bool builtin_signbit(T arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return std::signbit(arg);
+ #else
+ return arg < T() || (arg == T() && T(1)/arg < T());
+ #endif
+ }
-/// \}
-/// \name Conversion
-/// \{
+ /// \}
+ /// \name Conversion
+ /// \{
-/// Convert IEEE single-precision to half-precision.
-/// Credit for this goes to [Jeroen van der
-/// Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf).
-/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
-/// rounding
-/// \param value single-precision value
-/// \return binary representation of half-precision value
-template <std::float_round_style R>
-uint16 float2half_impl(float value, true_type) {
- typedef bits<float>::type uint32;
- uint32
- bits; // = *reinterpret_cast<uint32*>(&value); //violating
- // strict aliasing!
- std::memcpy(&bits, &value, sizeof(float));
- /* uint16 hbits = (bits>>16) & 0x8000;
+ /// Convert IEEE single-precision to half-precision.
+ /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf).
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \param value single-precision value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R> uint16 float2half_impl(float value, true_type)
+ {
+ typedef bits<float>::type uint32;
+ uint32 bits;// = *reinterpret_cast<uint32*>(&value); //violating strict aliasing!
+ std::memcpy(&bits, &value, sizeof(float));
+/* uint16 hbits = (bits>>16) & 0x8000;
bits &= 0x7FFFFFFF;
int exp = bits >> 23;
if(exp == 255)
@@ -615,310 +498,254 @@ uint16 float2half_impl(float value, true_type) { hbits += ~(hbits>>15) & (s|g);
else if(R == std::round_toward_neg_infinity)
hbits += (hbits>>15) & (g|s);
-*/ static const uint16
- base_table[512] = {
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001,
- 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100,
- 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00,
- 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00,
- 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, 0x5C00,
- 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001,
- 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100,
- 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00,
- 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00,
- 0xC000, 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00,
- 0xE000, 0xE400, 0xE800, 0xEC00, 0xF000, 0xF400, 0xF800, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00};
- static const unsigned char shift_table[512] = {
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19,
- 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
- 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 23,
- 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13,
- 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
- 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 13};
- uint16 hbits =
- base_table[bits >> 23] +
- static_cast<uint16>((bits & 0x7FFFFF) >> shift_table[bits >> 23]);
- if (R == std::round_to_nearest)
- hbits +=
- (((bits & 0x7FFFFF) >> (shift_table[bits >> 23] - 1)) |
- (((bits >> 23) & 0xFF) == 102)) &
- ((hbits & 0x7C00) != 0x7C00)
-#if HALF_ROUND_TIES_TO_EVEN
- & (((((static_cast<uint32>(1) << (shift_table[bits >> 23] - 1)) - 1) &
- bits) != 0) |
- hbits)
-#endif
- ;
- else if (R == std::round_toward_zero)
- hbits -= ((hbits & 0x7FFF) == 0x7C00) & ~shift_table[bits >> 23];
- else if (R == std::round_toward_infinity)
- hbits +=
- ((((bits & 0x7FFFFF &
- ((static_cast<uint32>(1) << (shift_table[bits >> 23])) - 1)) != 0) |
- (((bits >> 23) <= 102) & ((bits >> 23) != 0))) &
- (hbits < 0x7C00)) -
- ((hbits == 0xFC00) & ((bits >> 23) != 511));
- else if (R == std::round_toward_neg_infinity)
- hbits +=
- ((((bits & 0x7FFFFF &
- ((static_cast<uint32>(1) << (shift_table[bits >> 23])) - 1)) != 0) |
- (((bits >> 23) <= 358) & ((bits >> 23) != 256))) &
- (hbits < 0xFC00) & (hbits >> 15)) -
- ((hbits == 0x7C00) & ((bits >> 23) != 255));
- return hbits;
-}
+*/ static const uint16 base_table[512] = {
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100,
+ 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00, 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00,
+ 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100,
+ 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00,
+ 0xC000, 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00, 0xF000, 0xF400, 0xF800, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00 };
+ static const unsigned char shift_table[512] = {
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
+ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
+ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13 };
+ uint16 hbits = base_table[bits>>23] + static_cast<uint16>((bits&0x7FFFFF)>>shift_table[bits>>23]);
+ if(R == std::round_to_nearest)
+ hbits += (((bits&0x7FFFFF)>>(shift_table[bits>>23]-1))|(((bits>>23)&0xFF)==102)) & ((hbits&0x7C00)!=0x7C00)
+ #if HALF_ROUND_TIES_TO_EVEN
+ & (((((static_cast<uint32>(1)<<(shift_table[bits>>23]-1))-1)&bits)!=0)|hbits)
+ #endif
+ ;
+ else if(R == std::round_toward_zero)
+ hbits -= ((hbits&0x7FFF)==0x7C00) & ~shift_table[bits>>23];
+ else if(R == std::round_toward_infinity)
+ hbits += ((((bits&0x7FFFFF&((static_cast<uint32>(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=102)&
+ ((bits>>23)!=0)))&(hbits<0x7C00)) - ((hbits==0xFC00)&((bits>>23)!=511));
+ else if(R == std::round_toward_neg_infinity)
+ hbits += ((((bits&0x7FFFFF&((static_cast<uint32>(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=358)&
+ ((bits>>23)!=256)))&(hbits<0xFC00)&(hbits>>15)) - ((hbits==0x7C00)&((bits>>23)!=255));
+ return hbits;
+ }
-/// Convert IEEE double-precision to half-precision.
-/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
-/// rounding
-/// \param value double-precision value
-/// \return binary representation of half-precision value
-template <std::float_round_style R>
-uint16 float2half_impl(double value, true_type) {
- typedef bits<float>::type uint32;
- typedef bits<double>::type uint64;
- uint64
- bits; // = *reinterpret_cast<uint64*>(&value); //violating
- // strict aliasing!
- std::memcpy(&bits, &value, sizeof(double));
- uint32 hi = bits >> 32, lo = bits & 0xFFFFFFFF;
- uint16 hbits = (hi >> 16) & 0x8000;
- hi &= 0x7FFFFFFF;
- int exp = hi >> 20;
- if (exp == 2047)
- return hbits | 0x7C00 |
- (0x3FF & -static_cast<unsigned>((bits & 0xFFFFFFFFFFFFF) != 0));
- if (exp > 1038) {
- if (R == std::round_toward_infinity) return hbits | 0x7C00 - (hbits >> 15);
- if (R == std::round_toward_neg_infinity)
- return hbits | 0x7BFF + (hbits >> 15);
- return hbits | 0x7BFF + (R != std::round_toward_zero);
- }
- int g, s = lo != 0;
- if (exp > 1008) {
- g = (hi >> 9) & 1;
- s |= (hi & 0x1FF) != 0;
- hbits |= ((exp - 1008) << 10) | ((hi >> 10) & 0x3FF);
- } else if (exp > 997) {
- int i = 1018 - exp;
- hi = (hi & 0xFFFFF) | 0x100000;
- g = (hi >> i) & 1;
- s |= (hi & ((1L << i) - 1)) != 0;
- hbits |= hi >> (i + 1);
- } else {
- g = 0;
- s |= hi != 0;
- }
- if (R == std::round_to_nearest)
-#if HALF_ROUND_TIES_TO_EVEN
- hbits += g & (s | hbits);
-#else
- hbits += g;
-#endif
- else if (R == std::round_toward_infinity)
- hbits += ~(hbits >> 15) & (s | g);
- else if (R == std::round_toward_neg_infinity)
- hbits += (hbits >> 15) & (g | s);
- return hbits;
-}
+ /// Convert IEEE double-precision to half-precision.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \param value double-precision value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R> uint16 float2half_impl(double value, true_type)
+ {
+ typedef bits<float>::type uint32;
+ typedef bits<double>::type uint64;
+ uint64 bits;// = *reinterpret_cast<uint64*>(&value); //violating strict aliasing!
+ std::memcpy(&bits, &value, sizeof(double));
+ uint32 hi = bits >> 32, lo = bits & 0xFFFFFFFF;
+ uint16 hbits = (hi>>16) & 0x8000;
+ hi &= 0x7FFFFFFF;
+ int exp = hi >> 20;
+ if(exp == 2047)
+ return hbits | 0x7C00 | (0x3FF&-static_cast<unsigned>((bits&0xFFFFFFFFFFFFF)!=0));
+ if(exp > 1038)
+ {
+ if(R == std::round_toward_infinity)
+ return hbits | 0x7C00 - (hbits>>15);
+ if(R == std::round_toward_neg_infinity)
+ return hbits | 0x7BFF + (hbits>>15);
+ return hbits | 0x7BFF + (R!=std::round_toward_zero);
+ }
+ int g, s = lo != 0;
+ if(exp > 1008)
+ {
+ g = (hi>>9) & 1;
+ s |= (hi&0x1FF) != 0;
+ hbits |= ((exp-1008)<<10) | ((hi>>10)&0x3FF);
+ }
+ else if(exp > 997)
+ {
+ int i = 1018 - exp;
+ hi = (hi&0xFFFFF) | 0x100000;
+ g = (hi>>i) & 1;
+ s |= (hi&((1L<<i)-1)) != 0;
+ hbits |= hi >> (i+1);
+ }
+ else
+ {
+ g = 0;
+ s |= hi != 0;
+ }
+ if(R == std::round_to_nearest)
+ #if HALF_ROUND_TIES_TO_EVEN
+ hbits += g & (s|hbits);
+ #else
+ hbits += g;
+ #endif
+ else if(R == std::round_toward_infinity)
+ hbits += ~(hbits>>15) & (s|g);
+ else if(R == std::round_toward_neg_infinity)
+ hbits += (hbits>>15) & (g|s);
+ return hbits;
+ }
-/// Convert non-IEEE floating point to half-precision.
-/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
-/// rounding
-/// \tparam T source type (builtin floating point type)
-/// \param value floating point value
-/// \return binary representation of half-precision value
-template <std::float_round_style R, typename T>
-uint16 float2half_impl(T value, ...) {
- uint16 hbits = static_cast<unsigned>(builtin_signbit(value)) << 15;
- if (value == T()) return hbits;
- if (builtin_isnan(value)) return hbits | 0x7FFF;
- if (builtin_isinf(value)) return hbits | 0x7C00;
- int exp;
- std::frexp(value, &exp);
- if (exp > 16) {
- if (R == std::round_toward_infinity)
- return hbits | (0x7C00 - (hbits >> 15));
- else if (R == std::round_toward_neg_infinity)
- return hbits | (0x7BFF + (hbits >> 15));
- return hbits | (0x7BFF + (R != std::round_toward_zero));
- }
- if (exp < -13)
- value = std::ldexp(value, 24);
- else {
- value = std::ldexp(value, 11 - exp);
- hbits |= ((exp + 13) << 10);
- }
- T ival, frac = std::modf(value, &ival);
- hbits += static_cast<uint16>(std::abs(static_cast<int>(ival)));
- if (R == std::round_to_nearest) {
- frac = std::abs(frac);
-#if HALF_ROUND_TIES_TO_EVEN
- hbits += (frac > T(0.5)) | ((frac == T(0.5)) & hbits);
-#else
- hbits += frac >= T(0.5);
-#endif
- } else if (R == std::round_toward_infinity)
- hbits += frac > T();
- else if (R == std::round_toward_neg_infinity)
- hbits += frac < T();
- return hbits;
-}
+ /// Convert non-IEEE floating point to half-precision.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T source type (builtin floating point type)
+ /// \param value floating point value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,typename T> uint16 float2half_impl(T value, ...)
+ {
+ uint16 hbits = static_cast<unsigned>(builtin_signbit(value)) << 15;
+ if(value == T())
+ return hbits;
+ if(builtin_isnan(value))
+ return hbits | 0x7FFF;
+ if(builtin_isinf(value))
+ return hbits | 0x7C00;
+ int exp;
+ std::frexp(value, &exp);
+ if(exp > 16)
+ {
+ if(R == std::round_toward_infinity)
+ return hbits | (0x7C00 - (hbits>>15));
+ else if(R == std::round_toward_neg_infinity)
+ return hbits | (0x7BFF + (hbits>>15));
+ return hbits | (0x7BFF + (R!=std::round_toward_zero));
+ }
+ if(exp < -13)
+ value = std::ldexp(value, 24);
+ else
+ {
+ value = std::ldexp(value, 11-exp);
+ hbits |= ((exp+13)<<10);
+ }
+ T ival, frac = std::modf(value, &ival);
+ hbits += static_cast<uint16>(std::abs(static_cast<int>(ival)));
+ if(R == std::round_to_nearest)
+ {
+ frac = std::abs(frac);
+ #if HALF_ROUND_TIES_TO_EVEN
+ hbits += (frac>T(0.5)) | ((frac==T(0.5))&hbits);
+ #else
+ hbits += frac >= T(0.5);
+ #endif
+ }
+ else if(R == std::round_toward_infinity)
+ hbits += frac > T();
+ else if(R == std::round_toward_neg_infinity)
+ hbits += frac < T();
+ return hbits;
+ }
-/// Convert floating point to half-precision.
-/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
-/// rounding
-/// \tparam T source type (builtin floating point type)
-/// \param value floating point value
-/// \return binary representation of half-precision value
-template <std::float_round_style R, typename T>
-uint16 float2half(T value) {
- return float2half_impl<R>(
- value, bool_type < std::numeric_limits<T>::is_iec559 &&
- sizeof(typename bits<T>::type) == sizeof(T) > ());
-}
+ /// Convert floating point to half-precision.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T source type (builtin floating point type)
+ /// \param value floating point value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,typename T> uint16 float2half(T value)
+ {
+ return float2half_impl<R>(value, bool_type<std::numeric_limits<T>::is_iec559&&sizeof(typename bits<T>::type)==sizeof(T)>());
+ }
-/// Convert integer to half-precision floating point.
-/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
-/// rounding
-/// \tparam S `true` if value negative, `false` else
-/// \tparam T type to convert (builtin integer type)
-/// \param value non-negative integral value
-/// \return binary representation of half-precision value
-template <std::float_round_style R, bool S, typename T>
-uint16 int2half_impl(T value) {
-#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
- static_assert(std::is_integral<T>::value,
- "int to half conversion only supports builtin integer types");
-#endif
- if (S) value = -value;
- uint16 bits = S << 15;
- if (value > 0xFFFF) {
- if (R == std::round_toward_infinity)
- bits |= 0x7C00 - S;
- else if (R == std::round_toward_neg_infinity)
- bits |= 0x7BFF + S;
- else
- bits |= 0x7BFF + (R != std::round_toward_zero);
- } else if (value) {
- unsigned int m = value, exp = 24;
- for (; m < 0x400; m <<= 1, --exp)
- ;
- for (; m > 0x7FF; m >>= 1, ++exp)
- ;
- bits |= (exp << 10) + m;
- if (exp > 24) {
- if (R == std::round_to_nearest)
- bits += (value >> (exp - 25)) & 1
-#if HALF_ROUND_TIES_TO_EVEN
- & (((((1 << (exp - 25)) - 1) & value) != 0) | bits)
-#endif
- ;
- else if (R == std::round_toward_infinity)
- bits += ((value & ((1 << (exp - 24)) - 1)) != 0) & !S;
- else if (R == std::round_toward_neg_infinity)
- bits += ((value & ((1 << (exp - 24)) - 1)) != 0) & S;
- }
- }
- return bits;
-}
+ /// Convert integer to half-precision floating point.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam S `true` if value negative, `false` else
+ /// \tparam T type to convert (builtin integer type)
+ /// \param value non-negative integral value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,bool S,typename T> uint16 int2half_impl(T value)
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_integral<T>::value, "int to half conversion only supports builtin integer types");
+ #endif
+ if(S)
+ value = -value;
+ uint16 bits = S << 15;
+ if(value > 0xFFFF)
+ {
+ if(R == std::round_toward_infinity)
+ bits |= 0x7C00 - S;
+ else if(R == std::round_toward_neg_infinity)
+ bits |= 0x7BFF + S;
+ else
+ bits |= 0x7BFF + (R!=std::round_toward_zero);
+ }
+ else if(value)
+ {
+ unsigned int m = value, exp = 24;
+ for(; m<0x400; m<<=1,--exp) ;
+ for(; m>0x7FF; m>>=1,++exp) ;
+ bits |= (exp<<10) + m;
+ if(exp > 24)
+ {
+ if(R == std::round_to_nearest)
+ bits += (value>>(exp-25)) & 1
+ #if HALF_ROUND_TIES_TO_EVEN
+ & (((((1<<(exp-25))-1)&value)!=0)|bits)
+ #endif
+ ;
+ else if(R == std::round_toward_infinity)
+ bits += ((value&((1<<(exp-24))-1))!=0) & !S;
+ else if(R == std::round_toward_neg_infinity)
+ bits += ((value&((1<<(exp-24))-1))!=0) & S;
+ }
+ }
+ return bits;
+ }
-/// Convert integer to half-precision floating point.
-/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
-/// rounding
-/// \tparam T type to convert (builtin integer type)
-/// \param value integral value
-/// \return binary representation of half-precision value
-template <std::float_round_style R, typename T>
-uint16 int2half(T value) {
- return (value < 0) ? int2half_impl<R, true>(value)
- : int2half_impl<R, false>(value);
-}
+ /// Convert integer to half-precision floating point.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T type to convert (builtin integer type)
+ /// \param value integral value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,typename T> uint16 int2half(T value)
+ {
+ return (value<0) ? int2half_impl<R,true>(value) : int2half_impl<R,false>(value);
+ }
-/// Convert half-precision to IEEE single-precision.
-/// Credit for this goes to [Jeroen van der
-/// Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf).
-/// \param value binary representation of half-precision value
-/// \return single-precision value
-inline float half2float_impl(uint16 value, float, true_type) {
- typedef bits<float>::type uint32;
- /* uint32 bits = static_cast<uint32>(value&0x8000) << 16;
+ /// Convert half-precision to IEEE single-precision.
+ /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf).
+ /// \param value binary representation of half-precision value
+ /// \return single-precision value
+ inline float half2float_impl(uint16 value, float, true_type)
+ {
+ typedef bits<float>::type uint32;
+/* uint32 bits = static_cast<uint32>(value&0x8000) << 16;
int abs = value & 0x7FFF;
if(abs)
{
@@ -926,2927 +753,2315 @@ inline float half2float_impl(uint16 value, float, true_type) { for(; abs<0x400; abs<<=1,bits-=0x800000) ;
bits += static_cast<uint32>(abs) << 13;
}
-*/ static const uint32
- mantissa_table[2048] = {
- 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000,
- 0x34A00000, 0x34C00000, 0x34E00000, 0x35000000, 0x35100000,
- 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000,
- 0x35700000, 0x35800000, 0x35880000, 0x35900000, 0x35980000,
- 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000,
- 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000,
- 0x35F00000, 0x35F80000, 0x36000000, 0x36040000, 0x36080000,
- 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000,
- 0x36200000, 0x36240000, 0x36280000, 0x362C0000, 0x36300000,
- 0x36340000, 0x36380000, 0x363C0000, 0x36400000, 0x36440000,
- 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000,
- 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000,
- 0x36700000, 0x36740000, 0x36780000, 0x367C0000, 0x36800000,
- 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000,
- 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000,
- 0x36960000, 0x36980000, 0x369A0000, 0x369C0000, 0x369E0000,
- 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, 0x36A80000,
- 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000,
- 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000,
- 0x36BE0000, 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000,
- 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000,
- 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000,
- 0x36DC0000, 0x36DE0000, 0x36E00000, 0x36E20000, 0x36E40000,
- 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000,
- 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000,
- 0x36FA0000, 0x36FC0000, 0x36FE0000, 0x37000000, 0x37010000,
- 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000,
- 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000,
- 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000, 0x37100000,
- 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000,
- 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000,
- 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000,
- 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000,
- 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000,
- 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000,
- 0x372F0000, 0x37300000, 0x37310000, 0x37320000, 0x37330000,
- 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000,
- 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000,
- 0x373E0000, 0x373F0000, 0x37400000, 0x37410000, 0x37420000,
- 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000,
- 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, 0x374C0000,
- 0x374D0000, 0x374E0000, 0x374F0000, 0x37500000, 0x37510000,
- 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000,
- 0x37570000, 0x37580000, 0x37590000, 0x375A0000, 0x375B0000,
- 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000, 0x37600000,
- 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000,
- 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376A0000,
- 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000,
- 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000,
- 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000,
- 0x377A0000, 0x377B0000, 0x377C0000, 0x377D0000, 0x377E0000,
- 0x377F0000, 0x37800000, 0x37808000, 0x37810000, 0x37818000,
- 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000,
- 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000,
- 0x37870000, 0x37878000, 0x37880000, 0x37888000, 0x37890000,
- 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000,
- 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000,
- 0x378E8000, 0x378F0000, 0x378F8000, 0x37900000, 0x37908000,
- 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000,
- 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000,
- 0x37960000, 0x37968000, 0x37970000, 0x37978000, 0x37980000,
- 0x37988000, 0x37990000, 0x37998000, 0x379A0000, 0x379A8000,
- 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000,
- 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000,
- 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, 0x37A20000,
- 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000,
- 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000,
- 0x37A78000, 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000,
- 0x37AA0000, 0x37AA8000, 0x37AB0000, 0x37AB8000, 0x37AC0000,
- 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000,
- 0x37AF0000, 0x37AF8000, 0x37B00000, 0x37B08000, 0x37B10000,
- 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000,
- 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000,
- 0x37B68000, 0x37B70000, 0x37B78000, 0x37B80000, 0x37B88000,
- 0x37B90000, 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000,
- 0x37BB8000, 0x37BC0000, 0x37BC8000, 0x37BD0000, 0x37BD8000,
- 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000, 0x37C00000,
- 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000,
- 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000,
- 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000,
- 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000,
- 0x37CA8000, 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000,
- 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, 0x37CF0000,
- 0x37CF8000, 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000,
- 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000,
- 0x37D48000, 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000,
- 0x37D70000, 0x37D78000, 0x37D80000, 0x37D88000, 0x37D90000,
- 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000,
- 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000,
- 0x37DE8000, 0x37DF0000, 0x37DF8000, 0x37E00000, 0x37E08000,
- 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000,
- 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000,
- 0x37E60000, 0x37E68000, 0x37E70000, 0x37E78000, 0x37E80000,
- 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, 0x37EA8000,
- 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000,
- 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000,
- 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000,
- 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000,
- 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000,
- 0x37F78000, 0x37F80000, 0x37F88000, 0x37F90000, 0x37F98000,
- 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, 0x37FC0000,
- 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000,
- 0x37FF0000, 0x37FF8000, 0x38000000, 0x38004000, 0x38008000,
- 0x3800C000, 0x38010000, 0x38014000, 0x38018000, 0x3801C000,
- 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000,
- 0x38034000, 0x38038000, 0x3803C000, 0x38040000, 0x38044000,
- 0x38048000, 0x3804C000, 0x38050000, 0x38054000, 0x38058000,
- 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000,
- 0x38070000, 0x38074000, 0x38078000, 0x3807C000, 0x38080000,
- 0x38084000, 0x38088000, 0x3808C000, 0x38090000, 0x38094000,
- 0x38098000, 0x3809C000, 0x380A0000, 0x380A4000, 0x380A8000,
- 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000,
- 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000,
- 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000,
- 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000,
- 0x380FC000, 0x38100000, 0x38104000, 0x38108000, 0x3810C000,
- 0x38110000, 0x38114000, 0x38118000, 0x3811C000, 0x38120000,
- 0x38124000, 0x38128000, 0x3812C000, 0x38130000, 0x38134000,
- 0x38138000, 0x3813C000, 0x38140000, 0x38144000, 0x38148000,
- 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000,
- 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000,
- 0x38174000, 0x38178000, 0x3817C000, 0x38180000, 0x38184000,
- 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000,
- 0x3819C000, 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000,
- 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000, 0x381C0000,
- 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000,
- 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000,
- 0x381EC000, 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000,
- 0x38200000, 0x38204000, 0x38208000, 0x3820C000, 0x38210000,
- 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000,
- 0x38228000, 0x3822C000, 0x38230000, 0x38234000, 0x38238000,
- 0x3823C000, 0x38240000, 0x38244000, 0x38248000, 0x3824C000,
- 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000,
- 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000,
- 0x38278000, 0x3827C000, 0x38280000, 0x38284000, 0x38288000,
- 0x3828C000, 0x38290000, 0x38294000, 0x38298000, 0x3829C000,
- 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000,
- 0x382B4000, 0x382B8000, 0x382BC000, 0x382C0000, 0x382C4000,
- 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000,
- 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000,
- 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000, 0x38300000,
- 0x38304000, 0x38308000, 0x3830C000, 0x38310000, 0x38314000,
- 0x38318000, 0x3831C000, 0x38320000, 0x38324000, 0x38328000,
- 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000,
- 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000,
- 0x38354000, 0x38358000, 0x3835C000, 0x38360000, 0x38364000,
- 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000,
- 0x3837C000, 0x38380000, 0x38384000, 0x38388000, 0x3838C000,
- 0x38390000, 0x38394000, 0x38398000, 0x3839C000, 0x383A0000,
- 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000,
- 0x383B8000, 0x383BC000, 0x383C0000, 0x383C4000, 0x383C8000,
- 0x383CC000, 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000,
- 0x383E0000, 0x383E4000, 0x383E8000, 0x383EC000, 0x383F0000,
- 0x383F4000, 0x383F8000, 0x383FC000, 0x38400000, 0x38404000,
- 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000,
- 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000,
- 0x38430000, 0x38434000, 0x38438000, 0x3843C000, 0x38440000,
- 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000,
- 0x38458000, 0x3845C000, 0x38460000, 0x38464000, 0x38468000,
- 0x3846C000, 0x38470000, 0x38474000, 0x38478000, 0x3847C000,
- 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000,
- 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000,
- 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000,
- 0x384BC000, 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000,
- 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000,
- 0x384E4000, 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000,
- 0x384F8000, 0x384FC000, 0x38500000, 0x38504000, 0x38508000,
- 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000,
- 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000,
- 0x38534000, 0x38538000, 0x3853C000, 0x38540000, 0x38544000,
- 0x38548000, 0x3854C000, 0x38550000, 0x38554000, 0x38558000,
- 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000,
- 0x38570000, 0x38574000, 0x38578000, 0x3857C000, 0x38580000,
- 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000,
- 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000,
- 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000,
- 0x385C0000, 0x385C4000, 0x385C8000, 0x385CC000, 0x385D0000,
- 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, 0x385E4000,
- 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000,
- 0x385FC000, 0x38600000, 0x38604000, 0x38608000, 0x3860C000,
- 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000,
- 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000,
- 0x38638000, 0x3863C000, 0x38640000, 0x38644000, 0x38648000,
- 0x3864C000, 0x38650000, 0x38654000, 0x38658000, 0x3865C000,
- 0x38660000, 0x38664000, 0x38668000, 0x3866C000, 0x38670000,
- 0x38674000, 0x38678000, 0x3867C000, 0x38680000, 0x38684000,
- 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000,
- 0x3869C000, 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000,
- 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000, 0x386C0000,
- 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000,
- 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, 0x386E8000,
- 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000,
- 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000,
- 0x38714000, 0x38718000, 0x3871C000, 0x38720000, 0x38724000,
- 0x38728000, 0x3872C000, 0x38730000, 0x38734000, 0x38738000,
- 0x3873C000, 0x38740000, 0x38744000, 0x38748000, 0x3874C000,
- 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000,
- 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000,
- 0x38778000, 0x3877C000, 0x38780000, 0x38784000, 0x38788000,
- 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000,
- 0x387A0000, 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000,
- 0x387B4000, 0x387B8000, 0x387BC000, 0x387C0000, 0x387C4000,
- 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000,
- 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000,
- 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000, 0x38000000,
- 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800A000,
- 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000,
- 0x38016000, 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000,
- 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000,
- 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000,
- 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000,
- 0x3803E000, 0x38040000, 0x38042000, 0x38044000, 0x38046000,
- 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, 0x38050000,
- 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000,
- 0x3805C000, 0x3805E000, 0x38060000, 0x38062000, 0x38064000,
- 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000,
- 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000,
- 0x3807A000, 0x3807C000, 0x3807E000, 0x38080000, 0x38082000,
- 0x38084000, 0x38086000, 0x38088000, 0x3808A000, 0x3808C000,
- 0x3808E000, 0x38090000, 0x38092000, 0x38094000, 0x38096000,
- 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000, 0x380A0000,
- 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000,
- 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000,
- 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000,
- 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000,
- 0x380CA000, 0x380CC000, 0x380CE000, 0x380D0000, 0x380D2000,
- 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, 0x380DC000,
- 0x380DE000, 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000,
- 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000,
- 0x380F2000, 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000,
- 0x380FC000, 0x380FE000, 0x38100000, 0x38102000, 0x38104000,
- 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000,
- 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000,
- 0x3811A000, 0x3811C000, 0x3811E000, 0x38120000, 0x38122000,
- 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000,
- 0x3812E000, 0x38130000, 0x38132000, 0x38134000, 0x38136000,
- 0x38138000, 0x3813A000, 0x3813C000, 0x3813E000, 0x38140000,
- 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000,
- 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000,
- 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000,
- 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000,
- 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000,
- 0x38174000, 0x38176000, 0x38178000, 0x3817A000, 0x3817C000,
- 0x3817E000, 0x38180000, 0x38182000, 0x38184000, 0x38186000,
- 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000,
- 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000,
- 0x3819C000, 0x3819E000, 0x381A0000, 0x381A2000, 0x381A4000,
- 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000,
- 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000,
- 0x381BA000, 0x381BC000, 0x381BE000, 0x381C0000, 0x381C2000,
- 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, 0x381CC000,
- 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000,
- 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000, 0x381E0000,
- 0x381E2000, 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000,
- 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, 0x381F4000,
- 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000,
- 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000,
- 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000,
- 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000,
- 0x3821E000, 0x38220000, 0x38222000, 0x38224000, 0x38226000,
- 0x38228000, 0x3822A000, 0x3822C000, 0x3822E000, 0x38230000,
- 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823A000,
- 0x3823C000, 0x3823E000, 0x38240000, 0x38242000, 0x38244000,
- 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000,
- 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000,
- 0x3825A000, 0x3825C000, 0x3825E000, 0x38260000, 0x38262000,
- 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000,
- 0x3826E000, 0x38270000, 0x38272000, 0x38274000, 0x38276000,
- 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000, 0x38280000,
- 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000,
- 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000,
- 0x38296000, 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000,
- 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000,
- 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000,
- 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, 0x382BC000,
- 0x382BE000, 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000,
- 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000,
- 0x382D2000, 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000,
- 0x382DC000, 0x382DE000, 0x382E0000, 0x382E2000, 0x382E4000,
- 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000,
- 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000,
- 0x382FA000, 0x382FC000, 0x382FE000, 0x38300000, 0x38302000,
- 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000,
- 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000,
- 0x38318000, 0x3831A000, 0x3831C000, 0x3831E000, 0x38320000,
- 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832A000,
- 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000,
- 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000,
- 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000,
- 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, 0x38352000,
- 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000,
- 0x3835E000, 0x38360000, 0x38362000, 0x38364000, 0x38366000,
- 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, 0x38370000,
- 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000,
- 0x3837C000, 0x3837E000, 0x38380000, 0x38382000, 0x38384000,
- 0x38386000, 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000,
- 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000,
- 0x3839A000, 0x3839C000, 0x3839E000, 0x383A0000, 0x383A2000,
- 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000,
- 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000,
- 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000, 0x383C0000,
- 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000,
- 0x383CC000, 0x383CE000, 0x383D0000, 0x383D2000, 0x383D4000,
- 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, 0x383DE000,
- 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000,
- 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000,
- 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000,
- 0x383FE000, 0x38400000, 0x38402000, 0x38404000, 0x38406000,
- 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000,
- 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841A000,
- 0x3841C000, 0x3841E000, 0x38420000, 0x38422000, 0x38424000,
- 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000,
- 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000,
- 0x3843A000, 0x3843C000, 0x3843E000, 0x38440000, 0x38442000,
- 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000,
- 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000,
- 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000, 0x38460000,
- 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000,
- 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000,
- 0x38476000, 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000,
- 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000,
- 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000,
- 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000,
- 0x3849E000, 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000,
- 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000,
- 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000,
- 0x384BC000, 0x384BE000, 0x384C0000, 0x384C2000, 0x384C4000,
- 0x384C6000, 0x384C8000, 0x384CA000, 0x384CC000, 0x384CE000,
- 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000,
- 0x384DA000, 0x384DC000, 0x384DE000, 0x384E0000, 0x384E2000,
- 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000,
- 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000,
- 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000, 0x38500000,
- 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850A000,
- 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, 0x38514000,
- 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000,
- 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000,
- 0x3852A000, 0x3852C000, 0x3852E000, 0x38530000, 0x38532000,
- 0x38534000, 0x38536000, 0x38538000, 0x3853A000, 0x3853C000,
- 0x3853E000, 0x38540000, 0x38542000, 0x38544000, 0x38546000,
- 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000,
- 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000,
- 0x3855C000, 0x3855E000, 0x38560000, 0x38562000, 0x38564000,
- 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000,
- 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000,
- 0x3857A000, 0x3857C000, 0x3857E000, 0x38580000, 0x38582000,
- 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000,
- 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000,
- 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000, 0x385A0000,
- 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000,
- 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000,
- 0x385B6000, 0x385B8000, 0x385BA000, 0x385BC000, 0x385BE000,
- 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, 0x385C8000,
- 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000,
- 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000,
- 0x385DE000, 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000,
- 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000,
- 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000,
- 0x385FC000, 0x385FE000, 0x38600000, 0x38602000, 0x38604000,
- 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000,
- 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000,
- 0x3861A000, 0x3861C000, 0x3861E000, 0x38620000, 0x38622000,
- 0x38624000, 0x38626000, 0x38628000, 0x3862A000, 0x3862C000,
- 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000,
- 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000, 0x38640000,
- 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000,
- 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000,
- 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000,
- 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000,
- 0x3866A000, 0x3866C000, 0x3866E000, 0x38670000, 0x38672000,
- 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000,
- 0x3867E000, 0x38680000, 0x38682000, 0x38684000, 0x38686000,
- 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000,
- 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869A000,
- 0x3869C000, 0x3869E000, 0x386A0000, 0x386A2000, 0x386A4000,
- 0x386A6000, 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000,
- 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, 0x386B8000,
- 0x386BA000, 0x386BC000, 0x386BE000, 0x386C0000, 0x386C2000,
- 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000,
- 0x386CE000, 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000,
- 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000, 0x386E0000,
- 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000,
- 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000,
- 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000,
- 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000,
- 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000,
- 0x38714000, 0x38716000, 0x38718000, 0x3871A000, 0x3871C000,
- 0x3871E000, 0x38720000, 0x38722000, 0x38724000, 0x38726000,
- 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000,
- 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000,
- 0x3873C000, 0x3873E000, 0x38740000, 0x38742000, 0x38744000,
- 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000,
- 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000,
- 0x3875A000, 0x3875C000, 0x3875E000, 0x38760000, 0x38762000,
- 0x38764000, 0x38766000, 0x38768000, 0x3876A000, 0x3876C000,
- 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000,
- 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000, 0x38780000,
- 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878A000,
- 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000,
- 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000,
- 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, 0x387A8000,
- 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000,
- 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000,
- 0x387BE000, 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000,
- 0x387C8000, 0x387CA000, 0x387CC000, 0x387CE000, 0x387D0000,
- 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000,
- 0x387DC000, 0x387DE000, 0x387E0000, 0x387E2000, 0x387E4000,
- 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000,
- 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000,
- 0x387FA000, 0x387FC000, 0x387FE000};
- static const uint32 exponent_table[64] = {
- 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000,
- 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000,
- 0x06000000, 0x06800000, 0x07000000, 0x07800000, 0x08000000, 0x08800000,
- 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000,
- 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000,
- 0x0F000000, 0x47800000, 0x80000000, 0x80800000, 0x81000000, 0x81800000,
- 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000,
- 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000,
- 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000,
- 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000,
- 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000};
- static const unsigned short offset_table[64] = {
- 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,
- 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,
- 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 0,
- 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,
- 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,
- 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024};
- uint32 bits = mantissa_table[offset_table[value >> 10] + (value & 0x3FF)] +
- exponent_table[value >> 10];
- // return *reinterpret_cast<float*>(&bits); //violating
- //strict aliasing!
- float out;
- std::memcpy(&out, &bits, sizeof(float));
- return out;
-}
+*/ static const uint32 mantissa_table[2048] = {
+ 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000, 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000, 0x35700000,
+ 0x35800000, 0x35880000, 0x35900000, 0x35980000, 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, 0x35F00000, 0x35F80000,
+ 0x36000000, 0x36040000, 0x36080000, 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000, 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000,
+ 0x36400000, 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000, 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, 0x36700000, 0x36740000, 0x36780000, 0x367C0000,
+ 0x36800000, 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000, 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369A0000, 0x369C0000, 0x369E0000,
+ 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000,
+ 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000, 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000,
+ 0x36E00000, 0x36E20000, 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000, 0x36FC0000, 0x36FE0000,
+ 0x37000000, 0x37010000, 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000, 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000,
+ 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000,
+ 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000, 0x372F0000,
+ 0x37300000, 0x37310000, 0x37320000, 0x37330000, 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, 0x373E0000, 0x373F0000,
+ 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000,
+ 0x37500000, 0x37510000, 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000,
+ 0x37600000, 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000,
+ 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000, 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000,
+ 0x37800000, 0x37808000, 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, 0x37870000, 0x37878000,
+ 0x37880000, 0x37888000, 0x37890000, 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, 0x378F8000,
+ 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, 0x37960000, 0x37968000, 0x37970000, 0x37978000,
+ 0x37980000, 0x37988000, 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000, 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000,
+ 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000,
+ 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000, 0x37AB0000, 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000, 0x37AF0000, 0x37AF8000,
+ 0x37B00000, 0x37B08000, 0x37B10000, 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, 0x37B68000, 0x37B70000, 0x37B78000,
+ 0x37B80000, 0x37B88000, 0x37B90000, 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000, 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000,
+ 0x37C00000, 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000,
+ 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, 0x37CF0000, 0x37CF8000,
+ 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000, 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000,
+ 0x37D80000, 0x37D88000, 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000, 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000,
+ 0x37E00000, 0x37E08000, 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000, 0x37E70000, 0x37E78000,
+ 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000, 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000,
+ 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000,
+ 0x37F80000, 0x37F88000, 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, 0x37FF0000, 0x37FF8000,
+ 0x38000000, 0x38004000, 0x38008000, 0x3800C000, 0x38010000, 0x38014000, 0x38018000, 0x3801C000, 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000, 0x38034000, 0x38038000, 0x3803C000,
+ 0x38040000, 0x38044000, 0x38048000, 0x3804C000, 0x38050000, 0x38054000, 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, 0x38070000, 0x38074000, 0x38078000, 0x3807C000,
+ 0x38080000, 0x38084000, 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000, 0x380A0000, 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000,
+ 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000, 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, 0x380FC000,
+ 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, 0x38114000, 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000, 0x38130000, 0x38134000, 0x38138000, 0x3813C000,
+ 0x38140000, 0x38144000, 0x38148000, 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, 0x38174000, 0x38178000, 0x3817C000,
+ 0x38180000, 0x38184000, 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000, 0x3819C000, 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000,
+ 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000, 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000,
+ 0x38200000, 0x38204000, 0x38208000, 0x3820C000, 0x38210000, 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000, 0x38228000, 0x3822C000, 0x38230000, 0x38234000, 0x38238000, 0x3823C000,
+ 0x38240000, 0x38244000, 0x38248000, 0x3824C000, 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000,
+ 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000, 0x38298000, 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000, 0x382B4000, 0x382B8000, 0x382BC000,
+ 0x382C0000, 0x382C4000, 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000,
+ 0x38300000, 0x38304000, 0x38308000, 0x3830C000, 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000, 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000,
+ 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000, 0x38358000, 0x3835C000, 0x38360000, 0x38364000, 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000, 0x3837C000,
+ 0x38380000, 0x38384000, 0x38388000, 0x3838C000, 0x38390000, 0x38394000, 0x38398000, 0x3839C000, 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, 0x383B8000, 0x383BC000,
+ 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000, 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000, 0x383E8000, 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000,
+ 0x38400000, 0x38404000, 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000, 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, 0x38430000, 0x38434000, 0x38438000, 0x3843C000,
+ 0x38440000, 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, 0x3845C000, 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000, 0x38478000, 0x3847C000,
+ 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000, 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000,
+ 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000,
+ 0x38500000, 0x38504000, 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000, 0x38538000, 0x3853C000,
+ 0x38540000, 0x38544000, 0x38548000, 0x3854C000, 0x38550000, 0x38554000, 0x38558000, 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000, 0x38570000, 0x38574000, 0x38578000, 0x3857C000,
+ 0x38580000, 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000,
+ 0x385C0000, 0x385C4000, 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000, 0x385FC000,
+ 0x38600000, 0x38604000, 0x38608000, 0x3860C000, 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000, 0x38638000, 0x3863C000,
+ 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, 0x38654000, 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000, 0x38670000, 0x38674000, 0x38678000, 0x3867C000,
+ 0x38680000, 0x38684000, 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000, 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000,
+ 0x386C0000, 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000,
+ 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000, 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000, 0x38730000, 0x38734000, 0x38738000, 0x3873C000,
+ 0x38740000, 0x38744000, 0x38748000, 0x3874C000, 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000, 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, 0x38778000, 0x3877C000,
+ 0x38780000, 0x38784000, 0x38788000, 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000,
+ 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000, 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000,
+ 0x38000000, 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800A000, 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000,
+ 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000,
+ 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, 0x38050000, 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000, 0x3805C000, 0x3805E000,
+ 0x38060000, 0x38062000, 0x38064000, 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807A000, 0x3807C000, 0x3807E000,
+ 0x38080000, 0x38082000, 0x38084000, 0x38086000, 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000, 0x38094000, 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000,
+ 0x380A0000, 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000,
+ 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, 0x380CE000, 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, 0x380DC000, 0x380DE000,
+ 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000, 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000,
+ 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000,
+ 0x38120000, 0x38122000, 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000, 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000, 0x3813C000, 0x3813E000,
+ 0x38140000, 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000, 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000, 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000,
+ 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, 0x38176000, 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000,
+ 0x38180000, 0x38182000, 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, 0x3819C000, 0x3819E000,
+ 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, 0x381BC000, 0x381BE000,
+ 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000,
+ 0x381E0000, 0x381E2000, 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000,
+ 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, 0x3821E000,
+ 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000, 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823A000, 0x3823C000, 0x3823E000,
+ 0x38240000, 0x38242000, 0x38244000, 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825A000, 0x3825C000, 0x3825E000,
+ 0x38260000, 0x38262000, 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, 0x38272000, 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000,
+ 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000,
+ 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000, 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, 0x382BC000, 0x382BE000,
+ 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000, 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000,
+ 0x382E0000, 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000, 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000, 0x382FA000, 0x382FC000, 0x382FE000,
+ 0x38300000, 0x38302000, 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831A000, 0x3831C000, 0x3831E000,
+ 0x38320000, 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000, 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000,
+ 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000,
+ 0x38360000, 0x38362000, 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, 0x3837C000, 0x3837E000,
+ 0x38380000, 0x38382000, 0x38384000, 0x38386000, 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000,
+ 0x383A0000, 0x383A2000, 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000,
+ 0x383C0000, 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000, 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, 0x383DE000,
+ 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000, 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, 0x383FE000,
+ 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000,
+ 0x38420000, 0x38422000, 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000, 0x3843C000, 0x3843E000,
+ 0x38440000, 0x38442000, 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000, 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000,
+ 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000,
+ 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000, 0x3849E000,
+ 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, 0x384BE000,
+ 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000, 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000, 0x384DA000, 0x384DC000, 0x384DE000,
+ 0x384E0000, 0x384E2000, 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000,
+ 0x38500000, 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000,
+ 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000, 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, 0x38538000, 0x3853A000, 0x3853C000, 0x3853E000,
+ 0x38540000, 0x38542000, 0x38544000, 0x38546000, 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, 0x3855C000, 0x3855E000,
+ 0x38560000, 0x38562000, 0x38564000, 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000,
+ 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000, 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000,
+ 0x385A0000, 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, 0x385BA000, 0x385BC000, 0x385BE000,
+ 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000,
+ 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000, 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000,
+ 0x38600000, 0x38602000, 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000, 0x3861C000, 0x3861E000,
+ 0x38620000, 0x38622000, 0x38624000, 0x38626000, 0x38628000, 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000,
+ 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000,
+ 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000, 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000, 0x3867E000,
+ 0x38680000, 0x38682000, 0x38684000, 0x38686000, 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869A000, 0x3869C000, 0x3869E000,
+ 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000,
+ 0x386C0000, 0x386C2000, 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000, 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000,
+ 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000, 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000,
+ 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000,
+ 0x38720000, 0x38722000, 0x38724000, 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, 0x3873C000, 0x3873E000,
+ 0x38740000, 0x38742000, 0x38744000, 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, 0x3875E000,
+ 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000, 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000,
+ 0x38780000, 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000,
+ 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000,
+ 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000, 0x387CC000, 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000, 0x387DC000, 0x387DE000,
+ 0x387E0000, 0x387E2000, 0x387E4000, 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, 0x387FA000, 0x387FC000, 0x387FE000 };
+ static const uint32 exponent_table[64] = {
+ 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, 0x06000000, 0x06800000, 0x07000000, 0x07800000,
+ 0x08000000, 0x08800000, 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, 0x0F000000, 0x47800000,
+ 0x80000000, 0x80800000, 0x81000000, 0x81800000, 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000,
+ 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000 };
+ static const unsigned short offset_table[64] = {
+ 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,
+ 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024 };
+ uint32 bits = mantissa_table[offset_table[value>>10]+(value&0x3FF)] + exponent_table[value>>10];
+// return *reinterpret_cast<float*>(&bits); //violating strict aliasing!
+ float out;
+ std::memcpy(&out, &bits, sizeof(float));
+ return out;
+ }
-/// Convert half-precision to IEEE double-precision.
-/// \param value binary representation of half-precision value
-/// \return double-precision value
-inline double half2float_impl(uint16 value, double, true_type) {
- typedef bits<float>::type uint32;
- typedef bits<double>::type uint64;
- uint32 hi = static_cast<uint32>(value & 0x8000) << 16;
- int abs = value & 0x7FFF;
- if (abs) {
- hi |= 0x3F000000 << static_cast<unsigned>(abs >= 0x7C00);
- for (; abs < 0x400; abs <<= 1, hi -= 0x100000)
- ;
- hi += static_cast<uint32>(abs) << 10;
- }
- uint64 bits = static_cast<uint64>(hi) << 32;
- // return *reinterpret_cast<double*>(&bits); //violating
- //strict aliasing!
- double out;
- std::memcpy(&out, &bits, sizeof(double));
- return out;
-}
+ /// Convert half-precision to IEEE double-precision.
+ /// \param value binary representation of half-precision value
+ /// \return double-precision value
+ inline double half2float_impl(uint16 value, double, true_type)
+ {
+ typedef bits<float>::type uint32;
+ typedef bits<double>::type uint64;
+ uint32 hi = static_cast<uint32>(value&0x8000) << 16;
+ int abs = value & 0x7FFF;
+ if(abs)
+ {
+ hi |= 0x3F000000 << static_cast<unsigned>(abs>=0x7C00);
+ for(; abs<0x400; abs<<=1,hi-=0x100000) ;
+ hi += static_cast<uint32>(abs) << 10;
+ }
+ uint64 bits = static_cast<uint64>(hi) << 32;
+// return *reinterpret_cast<double*>(&bits); //violating strict aliasing!
+ double out;
+ std::memcpy(&out, &bits, sizeof(double));
+ return out;
+ }
-/// Convert half-precision to non-IEEE floating point.
-/// \tparam T type to convert to (builtin integer type)
-/// \param value binary representation of half-precision value
-/// \return floating point value
-template <typename T>
-T half2float_impl(uint16 value, T, ...) {
- T out;
- int abs = value & 0x7FFF;
- if (abs > 0x7C00)
- out = std::numeric_limits<T>::has_quiet_NaN
- ? std::numeric_limits<T>::quiet_NaN()
- : T();
- else if (abs == 0x7C00)
- out = std::numeric_limits<T>::has_infinity
- ? std::numeric_limits<T>::infinity()
- : std::numeric_limits<T>::max();
- else if (abs > 0x3FF)
- out = std::ldexp(static_cast<T>((abs & 0x3FF) | 0x400), (abs >> 10) - 25);
- else
- out = std::ldexp(static_cast<T>(abs), -24);
- return (value & 0x8000) ? -out : out;
-}
+ /// Convert half-precision to non-IEEE floating point.
+ /// \tparam T type to convert to (builtin integer type)
+ /// \param value binary representation of half-precision value
+ /// \return floating point value
+ template<typename T> T half2float_impl(uint16 value, T, ...)
+ {
+ T out;
+ int abs = value & 0x7FFF;
+ if(abs > 0x7C00)
+ out = std::numeric_limits<T>::has_quiet_NaN ? std::numeric_limits<T>::quiet_NaN() : T();
+ else if(abs == 0x7C00)
+ out = std::numeric_limits<T>::has_infinity ? std::numeric_limits<T>::infinity() : std::numeric_limits<T>::max();
+ else if(abs > 0x3FF)
+ out = std::ldexp(static_cast<T>((abs&0x3FF)|0x400), (abs>>10)-25);
+ else
+ out = std::ldexp(static_cast<T>(abs), -24);
+ return (value&0x8000) ? -out : out;
+ }
-/// Convert half-precision to floating point.
-/// \tparam T type to convert to (builtin integer type)
-/// \param value binary representation of half-precision value
-/// \return floating point value
-template <typename T>
-T half2float(uint16 value) {
- return half2float_impl(value, T(),
- bool_type < std::numeric_limits<T>::is_iec559 &&
- sizeof(typename bits<T>::type) == sizeof(T) > ());
-}
+ /// Convert half-precision to floating point.
+ /// \tparam T type to convert to (builtin integer type)
+ /// \param value binary representation of half-precision value
+ /// \return floating point value
+ template<typename T> T half2float(uint16 value)
+ {
+ return half2float_impl(value, T(), bool_type<std::numeric_limits<T>::is_iec559&&sizeof(typename bits<T>::type)==sizeof(T)>());
+ }
-/// Convert half-precision floating point to integer.
-/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
-/// rounding
-/// \tparam E `true` for round to even, `false` for round away from zero
-/// \tparam T type to convert to (buitlin integer type with at least 16 bits
-/// precision, excluding any implicit sign bits)
-/// \param value binary representation of half-precision value
-/// \return integral value
-template <std::float_round_style R, bool E, typename T>
-T half2int_impl(uint16 value) {
-#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
- static_assert(std::is_integral<T>::value,
- "half to int conversion only supports builtin integer types");
-#endif
- unsigned int e = value & 0x7FFF;
- if (e >= 0x7C00)
- return (value & 0x8000) ? std::numeric_limits<T>::min()
- : std::numeric_limits<T>::max();
- if (e < 0x3800) {
- if (R == std::round_toward_infinity)
- return T(~(value >> 15) & (e != 0));
- else if (R == std::round_toward_neg_infinity)
- return -T(value > 0x8000);
- return T();
- }
- unsigned int m = (value & 0x3FF) | 0x400;
- e >>= 10;
- if (e < 25) {
- if (R == std::round_to_nearest)
- m += (1 << (24 - e)) - (~(m >> (25 - e)) & E);
- else if (R == std::round_toward_infinity)
- m += ((value >> 15) - 1) & ((1 << (25 - e)) - 1U);
- else if (R == std::round_toward_neg_infinity)
- m += -(value >> 15) & ((1 << (25 - e)) - 1U);
- m >>= 25 - e;
- } else
- m <<= e - 25;
- return (value & 0x8000) ? -static_cast<T>(m) : static_cast<T>(m);
-}
-
-/// Convert half-precision floating point to integer.
-/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
-/// rounding
-/// \tparam T type to convert to (buitlin integer type with at least 16 bits
-/// precision, excluding any implicit sign bits)
-/// \param value binary representation of half-precision value
-/// \return integral value
-template <std::float_round_style R, typename T>
-T half2int(uint16 value) {
- return half2int_impl<R, HALF_ROUND_TIES_TO_EVEN, T>(value);
-}
+ /// Convert half-precision floating point to integer.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam E `true` for round to even, `false` for round away from zero
+ /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
+ /// \param value binary representation of half-precision value
+ /// \return integral value
+ template<std::float_round_style R,bool E,typename T> T half2int_impl(uint16 value)
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_integral<T>::value, "half to int conversion only supports builtin integer types");
+ #endif
+ unsigned int e = value & 0x7FFF;
+ if(e >= 0x7C00)
+ return (value&0x8000) ? std::numeric_limits<T>::min() : std::numeric_limits<T>::max();
+ if(e < 0x3800)
+ {
+ if(R == std::round_toward_infinity)
+ return T(~(value>>15)&(e!=0));
+ else if(R == std::round_toward_neg_infinity)
+ return -T(value>0x8000);
+ return T();
+ }
+ unsigned int m = (value&0x3FF) | 0x400;
+ e >>= 10;
+ if(e < 25)
+ {
+ if(R == std::round_to_nearest)
+ m += (1<<(24-e)) - (~(m>>(25-e))&E);
+ else if(R == std::round_toward_infinity)
+ m += ((value>>15)-1) & ((1<<(25-e))-1U);
+ else if(R == std::round_toward_neg_infinity)
+ m += -(value>>15) & ((1<<(25-e))-1U);
+ m >>= 25 - e;
+ }
+ else
+ m <<= e - 25;
+ return (value&0x8000) ? -static_cast<T>(m) : static_cast<T>(m);
+ }
-/// Convert half-precision floating point to integer using
-/// round-to-nearest-away-from-zero.
-/// \tparam T type to convert to (buitlin integer type with at least 16 bits
-/// precision, excluding any implicit sign bits)
-/// \param value binary representation of half-precision value
-/// \return integral value
-template <typename T>
-T half2int_up(uint16 value) {
- return half2int_impl<std::round_to_nearest, 0, T>(value);
-}
+ /// Convert half-precision floating point to integer.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
+ /// \param value binary representation of half-precision value
+ /// \return integral value
+ template<std::float_round_style R,typename T> T half2int(uint16 value) { return half2int_impl<R,HALF_ROUND_TIES_TO_EVEN,T>(value); }
-/// Round half-precision number to nearest integer value.
-/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
-/// rounding
-/// \tparam E `true` for round to even, `false` for round away from zero
-/// \param value binary representation of half-precision value
-/// \return half-precision bits for nearest integral value
-template <std::float_round_style R, bool E>
-uint16 round_half_impl(uint16 value) {
- unsigned int e = value & 0x7FFF;
- uint16 result = value;
- if (e < 0x3C00) {
- result &= 0x8000;
- if (R == std::round_to_nearest)
- result |= 0x3C00U & -(e >= (0x3800 + E));
- else if (R == std::round_toward_infinity)
- result |= 0x3C00U & -(~(value >> 15) & (e != 0));
- else if (R == std::round_toward_neg_infinity)
- result |= 0x3C00U & -(value > 0x8000);
- } else if (e < 0x6400) {
- e = 25 - (e >> 10);
- unsigned int mask = (1 << e) - 1;
- if (R == std::round_to_nearest)
- result += (1 << (e - 1)) - (~(result >> e) & E);
- else if (R == std::round_toward_infinity)
- result += mask & ((value >> 15) - 1);
- else if (R == std::round_toward_neg_infinity)
- result += mask & -(value >> 15);
- result &= ~mask;
- }
- return result;
-}
+ /// Convert half-precision floating point to integer using round-to-nearest-away-from-zero.
+ /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
+ /// \param value binary representation of half-precision value
+ /// \return integral value
+ template<typename T> T half2int_up(uint16 value) { return half2int_impl<std::round_to_nearest,0,T>(value); }
-/// Round half-precision number to nearest integer value.
-/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
-/// rounding
-/// \param value binary representation of half-precision value
-/// \return half-precision bits for nearest integral value
-template <std::float_round_style R>
-uint16 round_half(uint16 value) {
- return round_half_impl<R, HALF_ROUND_TIES_TO_EVEN>(value);
-}
-
-/// Round half-precision number to nearest integer value using
-/// round-to-nearest-away-from-zero.
-/// \param value binary representation of half-precision value
-/// \return half-precision bits for nearest integral value
-inline uint16 round_half_up(uint16 value) {
- return round_half_impl<std::round_to_nearest, 0>(value);
-}
-/// \}
+ /// Round half-precision number to nearest integer value.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam E `true` for round to even, `false` for round away from zero
+ /// \param value binary representation of half-precision value
+ /// \return half-precision bits for nearest integral value
+ template<std::float_round_style R,bool E> uint16 round_half_impl(uint16 value)
+ {
+ unsigned int e = value & 0x7FFF;
+ uint16 result = value;
+ if(e < 0x3C00)
+ {
+ result &= 0x8000;
+ if(R == std::round_to_nearest)
+ result |= 0x3C00U & -(e>=(0x3800+E));
+ else if(R == std::round_toward_infinity)
+ result |= 0x3C00U & -(~(value>>15)&(e!=0));
+ else if(R == std::round_toward_neg_infinity)
+ result |= 0x3C00U & -(value>0x8000);
+ }
+ else if(e < 0x6400)
+ {
+ e = 25 - (e>>10);
+ unsigned int mask = (1<<e) - 1;
+ if(R == std::round_to_nearest)
+ result += (1<<(e-1)) - (~(result>>e)&E);
+ else if(R == std::round_toward_infinity)
+ result += mask & ((value>>15)-1);
+ else if(R == std::round_toward_neg_infinity)
+ result += mask & -(value>>15);
+ result &= ~mask;
+ }
+ return result;
+ }
-struct functions;
-template <typename>
-struct unary_specialized;
-template <typename, typename>
-struct binary_specialized;
-template <typename, typename, std::float_round_style>
-struct half_caster;
-}
+ /// Round half-precision number to nearest integer value.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \param value binary representation of half-precision value
+ /// \return half-precision bits for nearest integral value
+ template<std::float_round_style R> uint16 round_half(uint16 value) { return round_half_impl<R,HALF_ROUND_TIES_TO_EVEN>(value); }
-/// Half-precision floating point type.
-/// This class implements an IEEE-conformant half-precision floating point type
-/// with the usual arithmetic operators and
-/// conversions. It is implicitly convertible to single-precision floating
-/// point, which makes artihmetic expressions and
-/// functions with mixed-type operands to be of the most precise operand type.
-/// Additionally all arithmetic operations
-/// (and many mathematical functions) are carried out in single-precision
-/// internally. All conversions from single- to
-/// half-precision are done using the library's default rounding mode, but
-/// temporary results inside chained arithmetic
-/// expressions are kept in single-precision as long as possible (while of
-/// course still maintaining a strong half-precision type).
-///
-/// According to the C++98/03 definition, the half type is not a POD type. But
-/// according to C++11's less strict and
-/// extended definitions it is both a standard layout type and a trivially
-/// copyable type (even if not a POD type), which
-/// means it can be standard-conformantly copied using raw binary copies. But in
-/// this context some more words about the
-/// actual size of the type. Although the half is representing an IEEE 16-bit
-/// type, it does not neccessarily have to be of
-/// exactly 16-bits size. But on any reasonable implementation the actual binary
-/// representation of this type will most
-/// probably not ivolve any additional "magic" or padding beyond the simple
-/// binary representation of the underlying 16-bit
-/// IEEE number, even if not strictly guaranteed by the standard. But even then
-/// it only has an actual size of 16 bits if
-/// your C++ implementation supports an unsigned integer type of exactly 16 bits
-/// width. But this should be the case on
-/// nearly any reasonable platform.
-///
-/// So if your C++ implementation is not totally exotic or imposes special
-/// alignment requirements, it is a reasonable
-/// assumption that the data of a half is just comprised of the 2 bytes of the
-/// underlying IEEE representation.
-class half {
- friend struct detail::functions;
- friend struct detail::unary_specialized<half>;
- friend struct detail::binary_specialized<half, half>;
- template <typename, typename, std::float_round_style>
- friend struct detail::half_caster;
- friend class std::numeric_limits<half>;
-#if HALF_ENABLE_CPP11_HASH
- friend struct std::hash<half>;
-#endif
-#if HALF_ENABLE_CPP11_USER_LITERALS
- friend half literal::operator"" _h(long double);
-#endif
+ /// Round half-precision number to nearest integer value using round-to-nearest-away-from-zero.
+ /// \param value binary representation of half-precision value
+ /// \return half-precision bits for nearest integral value
+ inline uint16 round_half_up(uint16 value) { return round_half_impl<std::round_to_nearest,0>(value); }
+ /// \}
- public:
- /// Default constructor.
- /// This initializes the half to 0. Although this does not match the builtin
- /// types' default-initialization semantics
- /// and may be less efficient than no initialization, it is needed to provide
- /// proper value-initialization semantics.
- HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {}
+ struct functions;
+ template<typename> struct unary_specialized;
+ template<typename,typename> struct binary_specialized;
+ template<typename,typename,std::float_round_style> struct half_caster;
+ }
- /// Copy constructor.
- /// \tparam T type of concrete half expression
- /// \param rhs half expression to copy from
- half(detail::expr rhs)
- : data_(detail::float2half<round_style>(static_cast<float>(rhs))) {}
+ /// Half-precision floating point type.
+ /// This class implements an IEEE-conformant half-precision floating point type with the usual arithmetic operators and
+ /// conversions. It is implicitly convertible to single-precision floating point, which makes artihmetic expressions and
+ /// functions with mixed-type operands to be of the most precise operand type. Additionally all arithmetic operations
+ /// (and many mathematical functions) are carried out in single-precision internally. All conversions from single- to
+ /// half-precision are done using the library's default rounding mode, but temporary results inside chained arithmetic
+ /// expressions are kept in single-precision as long as possible (while of course still maintaining a strong half-precision type).
+ ///
+ /// According to the C++98/03 definition, the half type is not a POD type. But according to C++11's less strict and
+ /// extended definitions it is both a standard layout type and a trivially copyable type (even if not a POD type), which
+ /// means it can be standard-conformantly copied using raw binary copies. But in this context some more words about the
+ /// actual size of the type. Although the half is representing an IEEE 16-bit type, it does not neccessarily have to be of
+ /// exactly 16-bits size. But on any reasonable implementation the actual binary representation of this type will most
+ /// probably not ivolve any additional "magic" or padding beyond the simple binary representation of the underlying 16-bit
+ /// IEEE number, even if not strictly guaranteed by the standard. But even then it only has an actual size of 16 bits if
+ /// your C++ implementation supports an unsigned integer type of exactly 16 bits width. But this should be the case on
+ /// nearly any reasonable platform.
+ ///
+ /// So if your C++ implementation is not totally exotic or imposes special alignment requirements, it is a reasonable
+ /// assumption that the data of a half is just comprised of the 2 bytes of the underlying IEEE representation.
+ class half
+ {
+ friend struct detail::functions;
+ friend struct detail::unary_specialized<half>;
+ friend struct detail::binary_specialized<half,half>;
+ template<typename,typename,std::float_round_style> friend struct detail::half_caster;
+ friend class std::numeric_limits<half>;
+ #if HALF_ENABLE_CPP11_HASH
+ friend struct std::hash<half>;
+ #endif
+ #if HALF_ENABLE_CPP11_USER_LITERALS
+ friend half literal::operator"" _h(long double);
+ #endif
- /// Conversion constructor.
- /// \param rhs float to convert
- explicit half(float rhs) : data_(detail::float2half<round_style>(rhs)) {}
+ public:
+ /// Default constructor.
+ /// This initializes the half to 0. Although this does not match the builtin types' default-initialization semantics
+ /// and may be less efficient than no initialization, it is needed to provide proper value-initialization semantics.
+ HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {}
- /// Conversion to single-precision.
- /// \return single precision value representing expression value
- operator float() const { return detail::half2float<float>(data_); }
+ /// Copy constructor.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to copy from
+ half(detail::expr rhs) : data_(detail::float2half<round_style>(static_cast<float>(rhs))) {}
- /// Assignment operator.
- /// \tparam T type of concrete half expression
- /// \param rhs half expression to copy from
- /// \return reference to this half
- half &operator=(detail::expr rhs) { return *this = static_cast<float>(rhs); }
+ /// Conversion constructor.
+ /// \param rhs float to convert
+ explicit half(float rhs) : data_(detail::float2half<round_style>(rhs)) {}
+
+ /// Conversion to single-precision.
+ /// \return single precision value representing expression value
+ operator float() const { return detail::half2float<float>(data_); }
- /// Arithmetic assignment.
- /// \tparam T type of concrete half expression
- /// \param rhs half expression to add
- /// \return reference to this half
- template <typename T>
- typename detail::enable<half &, T>::type operator+=(T rhs) {
- return *this += static_cast<float>(rhs);
- }
+ /// Assignment operator.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to copy from
+ /// \return reference to this half
+ half& operator=(detail::expr rhs) { return *this = static_cast<float>(rhs); }
- /// Arithmetic assignment.
- /// \tparam T type of concrete half expression
- /// \param rhs half expression to subtract
- /// \return reference to this half
- template <typename T>
- typename detail::enable<half &, T>::type operator-=(T rhs) {
- return *this -= static_cast<float>(rhs);
- }
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to add
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator+=(T rhs) { return *this += static_cast<float>(rhs); }
- /// Arithmetic assignment.
- /// \tparam T type of concrete half expression
- /// \param rhs half expression to multiply with
- /// \return reference to this half
- template <typename T>
- typename detail::enable<half &, T>::type operator*=(T rhs) {
- return *this *= static_cast<float>(rhs);
- }
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to subtract
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator-=(T rhs) { return *this -= static_cast<float>(rhs); }
- /// Arithmetic assignment.
- /// \tparam T type of concrete half expression
- /// \param rhs half expression to divide by
- /// \return reference to this half
- template <typename T>
- typename detail::enable<half &, T>::type operator/=(T rhs) {
- return *this /= static_cast<float>(rhs);
- }
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to multiply with
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator*=(T rhs) { return *this *= static_cast<float>(rhs); }
- /// Assignment operator.
- /// \param rhs single-precision value to copy from
- /// \return reference to this half
- half &operator=(float rhs) {
- data_ = detail::float2half<round_style>(rhs);
- return *this;
- }
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to divide by
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator/=(T rhs) { return *this /= static_cast<float>(rhs); }
- /// Arithmetic assignment.
- /// \param rhs single-precision value to add
- /// \return reference to this half
- half &operator+=(float rhs) {
- data_ =
- detail::float2half<round_style>(detail::half2float<float>(data_) + rhs);
- return *this;
- }
+ /// Assignment operator.
+ /// \param rhs single-precision value to copy from
+ /// \return reference to this half
+ half& operator=(float rhs) { data_ = detail::float2half<round_style>(rhs); return *this; }
- /// Arithmetic assignment.
- /// \param rhs single-precision value to subtract
- /// \return reference to this half
- half &operator-=(float rhs) {
- data_ =
- detail::float2half<round_style>(detail::half2float<float>(data_) - rhs);
- return *this;
- }
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to add
+ /// \return reference to this half
+ half& operator+=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)+rhs); return *this; }
- /// Arithmetic assignment.
- /// \param rhs single-precision value to multiply with
- /// \return reference to this half
- half &operator*=(float rhs) {
- data_ =
- detail::float2half<round_style>(detail::half2float<float>(data_) * rhs);
- return *this;
- }
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to subtract
+ /// \return reference to this half
+ half& operator-=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)-rhs); return *this; }
- /// Arithmetic assignment.
- /// \param rhs single-precision value to divide by
- /// \return reference to this half
- half &operator/=(float rhs) {
- data_ =
- detail::float2half<round_style>(detail::half2float<float>(data_) / rhs);
- return *this;
- }
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to multiply with
+ /// \return reference to this half
+ half& operator*=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)*rhs); return *this; }
- /// Prefix increment.
- /// \return incremented half value
- half &operator++() { return *this += 1.0f; }
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to divide by
+ /// \return reference to this half
+ half& operator/=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)/rhs); return *this; }
- /// Prefix decrement.
- /// \return decremented half value
- half &operator--() { return *this -= 1.0f; }
+ /// Prefix increment.
+ /// \return incremented half value
+ half& operator++() { return *this += 1.0f; }
- /// Postfix increment.
- /// \return non-incremented half value
- half operator++(int) {
- half out(*this);
- ++*this;
- return out;
- }
+ /// Prefix decrement.
+ /// \return decremented half value
+ half& operator--() { return *this -= 1.0f; }
- /// Postfix decrement.
- /// \return non-decremented half value
- half operator--(int) {
- half out(*this);
- --*this;
- return out;
- }
+ /// Postfix increment.
+ /// \return non-incremented half value
+ half operator++(int) { half out(*this); ++*this; return out; }
- private:
- /// Rounding mode to use
- static const std::float_round_style round_style =
- (std::float_round_style)(HALF_ROUND_STYLE);
+ /// Postfix decrement.
+ /// \return non-decremented half value
+ half operator--(int) { half out(*this); --*this; return out; }
+
+ private:
+ /// Rounding mode to use
+ static const std::float_round_style round_style = (std::float_round_style)(HALF_ROUND_STYLE);
- /// Constructor.
- /// \param bits binary representation to set half to
- HALF_CONSTEXPR half(detail::binary_t, detail::uint16 bits) HALF_NOEXCEPT
- : data_(bits) {}
+ /// Constructor.
+ /// \param bits binary representation to set half to
+ HALF_CONSTEXPR half(detail::binary_t, detail::uint16 bits) HALF_NOEXCEPT : data_(bits) {}
- /// Internal binary representation
- detail::uint16 data_;
-};
+ /// Internal binary representation
+ detail::uint16 data_;
+ };
#if HALF_ENABLE_CPP11_USER_LITERALS
-namespace literal {
-/// Half literal.
-/// While this returns an actual half-precision value, half literals can
-/// unfortunately not be constant expressions due
-/// to rather involved conversions.
-/// \param value literal value
-/// \return half with given value (if representable)
-inline half operator"" _h(long double value) {
- return half(detail::binary, detail::float2half<half::round_style>(value));
-}
-}
+ namespace literal
+ {
+ /// Half literal.
+ /// While this returns an actual half-precision value, half literals can unfortunately not be constant expressions due
+ /// to rather involved conversions.
+ /// \param value literal value
+ /// \return half with given value (if representable)
+ inline half operator"" _h(long double value) { return half(detail::binary, detail::float2half<half::round_style>(value)); }
+ }
#endif
-namespace detail {
-/// Wrapper implementing unspecialized half-precision functions.
-struct functions {
- /// Addition implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return Half-precision sum stored in single-precision
- static expr plus(float x, float y) { return expr(x + y); }
+ namespace detail
+ {
+ /// Wrapper implementing unspecialized half-precision functions.
+ struct functions
+ {
+ /// Addition implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision sum stored in single-precision
+ static expr plus(float x, float y) { return expr(x+y); }
- /// Subtraction implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return Half-precision difference stored in single-precision
- static expr minus(float x, float y) { return expr(x - y); }
+ /// Subtraction implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision difference stored in single-precision
+ static expr minus(float x, float y) { return expr(x-y); }
- /// Multiplication implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return Half-precision product stored in single-precision
- static expr multiplies(float x, float y) { return expr(x * y); }
+ /// Multiplication implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision product stored in single-precision
+ static expr multiplies(float x, float y) { return expr(x*y); }
- /// Division implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return Half-precision quotient stored in single-precision
- static expr divides(float x, float y) { return expr(x / y); }
+ /// Division implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision quotient stored in single-precision
+ static expr divides(float x, float y) { return expr(x/y); }
- /// Output implementation.
- /// \param out stream to write to
- /// \param arg value to write
- /// \return reference to stream
- template <typename charT, typename traits>
- static std::basic_ostream<charT, traits> &write(
- std::basic_ostream<charT, traits> &out, float arg) {
- return out << arg;
- }
+ /// Output implementation.
+ /// \param out stream to write to
+ /// \param arg value to write
+ /// \return reference to stream
+ template<typename charT,typename traits> static std::basic_ostream<charT,traits>& write(std::basic_ostream<charT,traits> &out, float arg) { return out << arg; }
- /// Input implementation.
- /// \param in stream to read from
- /// \param arg half to read into
- /// \return reference to stream
- template <typename charT, typename traits>
- static std::basic_istream<charT, traits> &read(
- std::basic_istream<charT, traits> &in, half &arg) {
- float f;
- if (in >> f) arg = f;
- return in;
- }
+ /// Input implementation.
+ /// \param in stream to read from
+ /// \param arg half to read into
+ /// \return reference to stream
+ template<typename charT,typename traits> static std::basic_istream<charT,traits>& read(std::basic_istream<charT,traits> &in, half &arg)
+ {
+ float f;
+ if(in >> f)
+ arg = f;
+ return in;
+ }
- /// Modulo implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return Half-precision division remainder stored in single-precision
- static expr fmod(float x, float y) { return expr(std::fmod(x, y)); }
+ /// Modulo implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision division remainder stored in single-precision
+ static expr fmod(float x, float y) { return expr(std::fmod(x, y)); }
- /// Remainder implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return Half-precision division remainder stored in single-precision
- static expr remainder(float x, float y) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::remainder(x, y));
-#else
- if (builtin_isnan(x) || builtin_isnan(y))
- return expr(std::numeric_limits<float>::quiet_NaN());
- float ax = std::fabs(x), ay = std::fabs(y);
- if (ax >= 65536.0f || ay < std::ldexp(1.0f, -24))
- return expr(std::numeric_limits<float>::quiet_NaN());
- if (ay >= 65536.0f) return expr(x);
- if (ax == ay) return expr(builtin_signbit(x) ? -0.0f : 0.0f);
- ax = std::fmod(ax, ay + ay);
- float y2 = 0.5f * ay;
- if (ax > y2) {
- ax -= ay;
- if (ax >= y2) ax -= ay;
- }
- return expr(builtin_signbit(x) ? -ax : ax);
-#endif
- }
+ /// Remainder implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision division remainder stored in single-precision
+ static expr remainder(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::remainder(x, y));
+ #else
+ if(builtin_isnan(x) || builtin_isnan(y))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ float ax = std::fabs(x), ay = std::fabs(y);
+ if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ if(ay >= 65536.0f)
+ return expr(x);
+ if(ax == ay)
+ return expr(builtin_signbit(x) ? -0.0f : 0.0f);
+ ax = std::fmod(ax, ay+ay);
+ float y2 = 0.5f * ay;
+ if(ax > y2)
+ {
+ ax -= ay;
+ if(ax >= y2)
+ ax -= ay;
+ }
+ return expr(builtin_signbit(x) ? -ax : ax);
+ #endif
+ }
- /// Remainder implementation.
- /// \param x first operand
- /// \param y second operand
- /// \param quo address to store quotient bits at
- /// \return Half-precision division remainder stored in single-precision
- static expr remquo(float x, float y, int *quo) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::remquo(x, y, quo));
-#else
- if (builtin_isnan(x) || builtin_isnan(y))
- return expr(std::numeric_limits<float>::quiet_NaN());
- bool sign = builtin_signbit(x),
- qsign = static_cast<bool>(sign ^ builtin_signbit(y));
- float ax = std::fabs(x), ay = std::fabs(y);
- if (ax >= 65536.0f || ay < std::ldexp(1.0f, -24))
- return expr(std::numeric_limits<float>::quiet_NaN());
- if (ay >= 65536.0f) return expr(x);
- if (ax == ay) return *quo = qsign ? -1 : 1, expr(sign ? -0.0f : 0.0f);
- ax = std::fmod(ax, 8.0f * ay);
- int cquo = 0;
- if (ax >= 4.0f * ay) {
- ax -= 4.0f * ay;
- cquo += 4;
- }
- if (ax >= 2.0f * ay) {
- ax -= 2.0f * ay;
- cquo += 2;
- }
- float y2 = 0.5f * ay;
- if (ax > y2) {
- ax -= ay;
- ++cquo;
- if (ax >= y2) {
- ax -= ay;
- ++cquo;
- }
- }
- return *quo = qsign ? -cquo : cquo, expr(sign ? -ax : ax);
-#endif
- }
+ /// Remainder implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param quo address to store quotient bits at
+ /// \return Half-precision division remainder stored in single-precision
+ static expr remquo(float x, float y, int *quo)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::remquo(x, y, quo));
+ #else
+ if(builtin_isnan(x) || builtin_isnan(y))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ bool sign = builtin_signbit(x), qsign = static_cast<bool>(sign^builtin_signbit(y));
+ float ax = std::fabs(x), ay = std::fabs(y);
+ if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ if(ay >= 65536.0f)
+ return expr(x);
+ if(ax == ay)
+ return *quo = qsign ? -1 : 1, expr(sign ? -0.0f : 0.0f);
+ ax = std::fmod(ax, 8.0f*ay);
+ int cquo = 0;
+ if(ax >= 4.0f * ay)
+ {
+ ax -= 4.0f * ay;
+ cquo += 4;
+ }
+ if(ax >= 2.0f * ay)
+ {
+ ax -= 2.0f * ay;
+ cquo += 2;
+ }
+ float y2 = 0.5f * ay;
+ if(ax > y2)
+ {
+ ax -= ay;
+ ++cquo;
+ if(ax >= y2)
+ {
+ ax -= ay;
+ ++cquo;
+ }
+ }
+ return *quo = qsign ? -cquo : cquo, expr(sign ? -ax : ax);
+ #endif
+ }
- /// Positive difference implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return Positive difference stored in single-precision
- static expr fdim(float x, float y) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::fdim(x, y));
-#else
- return expr((x <= y) ? 0.0f : (x - y));
-#endif
- }
+ /// Positive difference implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Positive difference stored in single-precision
+ static expr fdim(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fdim(x, y));
+ #else
+ return expr((x<=y) ? 0.0f : (x-y));
+ #endif
+ }
- /// Fused multiply-add implementation.
- /// \param x first operand
- /// \param y second operand
- /// \param z third operand
- /// \return \a x * \a y + \a z stored in single-precision
- static expr fma(float x, float y, float z) {
-#if HALF_ENABLE_CPP11_CMATH && defined(FP_FAST_FMAF)
- return expr(std::fma(x, y, z));
-#else
- return expr(x * y + z);
-#endif
- }
+ /// Fused multiply-add implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param z third operand
+ /// \return \a x * \a y + \a z stored in single-precision
+ static expr fma(float x, float y, float z)
+ {
+ #if HALF_ENABLE_CPP11_CMATH && defined(FP_FAST_FMAF)
+ return expr(std::fma(x, y, z));
+ #else
+ return expr(x*y+z);
+ #endif
+ }
- /// Get NaN.
- /// \return Half-precision quiet NaN
- static half nanh() { return half(binary, 0x7FFF); }
+ /// Get NaN.
+ /// \return Half-precision quiet NaN
+ static half nanh() { return half(binary, 0x7FFF); }
- /// Exponential implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr exp(float arg) { return expr(std::exp(arg)); }
+ /// Exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr exp(float arg) { return expr(std::exp(arg)); }
- /// Exponential implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr expm1(float arg) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::expm1(arg));
-#else
- return expr(static_cast<float>(std::exp(static_cast<double>(arg)) - 1.0));
-#endif
- }
+ /// Exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr expm1(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::expm1(arg));
+ #else
+ return expr(static_cast<float>(std::exp(static_cast<double>(arg))-1.0));
+ #endif
+ }
- /// Binary exponential implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr exp2(float arg) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::exp2(arg));
-#else
- return expr(
- static_cast<float>(std::exp(arg * 0.69314718055994530941723212145818)));
-#endif
- }
+ /// Binary exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr exp2(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::exp2(arg));
+ #else
+ return expr(static_cast<float>(std::exp(arg*0.69314718055994530941723212145818)));
+ #endif
+ }
- /// Logarithm implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr log(float arg) { return expr(std::log(arg)); }
+ /// Logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log(float arg) { return expr(std::log(arg)); }
- /// Common logarithm implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr log10(float arg) { return expr(std::log10(arg)); }
+ /// Common logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log10(float arg) { return expr(std::log10(arg)); }
- /// Logarithm implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr log1p(float arg) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::log1p(arg));
-#else
- return expr(static_cast<float>(std::log(1.0 + arg)));
-#endif
- }
+ /// Logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log1p(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::log1p(arg));
+ #else
+ return expr(static_cast<float>(std::log(1.0+arg)));
+ #endif
+ }
- /// Binary logarithm implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr log2(float arg) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::log2(arg));
-#else
- return expr(static_cast<float>(std::log(static_cast<double>(arg)) *
- 1.4426950408889634073599246810019));
-#endif
- }
+ /// Binary logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log2(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::log2(arg));
+ #else
+ return expr(static_cast<float>(std::log(static_cast<double>(arg))*1.4426950408889634073599246810019));
+ #endif
+ }
- /// Square root implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr sqrt(float arg) { return expr(std::sqrt(arg)); }
+ /// Square root implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sqrt(float arg) { return expr(std::sqrt(arg)); }
- /// Cubic root implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr cbrt(float arg) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::cbrt(arg));
-#else
- if (builtin_isnan(arg) || builtin_isinf(arg)) return expr(arg);
- return expr(builtin_signbit(arg)
- ? -static_cast<float>(
- std::pow(-static_cast<double>(arg), 1.0 / 3.0))
- : static_cast<float>(
- std::pow(static_cast<double>(arg), 1.0 / 3.0)));
-#endif
- }
+ /// Cubic root implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cbrt(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::cbrt(arg));
+ #else
+ if(builtin_isnan(arg) || builtin_isinf(arg))
+ return expr(arg);
+ return expr(builtin_signbit(arg) ? -static_cast<float>(std::pow(-static_cast<double>(arg), 1.0/3.0)) :
+ static_cast<float>(std::pow(static_cast<double>(arg), 1.0/3.0)));
+ #endif
+ }
- /// Hypotenuse implementation.
- /// \param x first argument
- /// \param y second argument
- /// \return function value stored in single-preicision
- static expr hypot(float x, float y) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::hypot(x, y));
-#else
- return expr(
- (builtin_isinf(x) || builtin_isinf(y))
- ? std::numeric_limits<float>::infinity()
- : static_cast<float>(std::sqrt(static_cast<double>(x) * x +
- static_cast<double>(y) * y)));
-#endif
- }
+ /// Hypotenuse implementation.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return function value stored in single-preicision
+ static expr hypot(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::hypot(x, y));
+ #else
+ return expr((builtin_isinf(x) || builtin_isinf(y)) ? std::numeric_limits<float>::infinity() :
+ static_cast<float>(std::sqrt(static_cast<double>(x)*x+static_cast<double>(y)*y)));
+ #endif
+ }
- /// Power implementation.
- /// \param base value to exponentiate
- /// \param exp power to expontiate to
- /// \return function value stored in single-preicision
- static expr pow(float base, float exp) { return expr(std::pow(base, exp)); }
+ /// Power implementation.
+ /// \param base value to exponentiate
+ /// \param exp power to expontiate to
+ /// \return function value stored in single-preicision
+ static expr pow(float base, float exp) { return expr(std::pow(base, exp)); }
- /// Sine implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr sin(float arg) { return expr(std::sin(arg)); }
+ /// Sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sin(float arg) { return expr(std::sin(arg)); }
- /// Cosine implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr cos(float arg) { return expr(std::cos(arg)); }
+ /// Cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cos(float arg) { return expr(std::cos(arg)); }
- /// Tan implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr tan(float arg) { return expr(std::tan(arg)); }
+ /// Tan implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tan(float arg) { return expr(std::tan(arg)); }
- /// Arc sine implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr asin(float arg) { return expr(std::asin(arg)); }
+ /// Arc sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr asin(float arg) { return expr(std::asin(arg)); }
- /// Arc cosine implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr acos(float arg) { return expr(std::acos(arg)); }
+ /// Arc cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr acos(float arg) { return expr(std::acos(arg)); }
- /// Arc tangent implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr atan(float arg) { return expr(std::atan(arg)); }
+ /// Arc tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr atan(float arg) { return expr(std::atan(arg)); }
- /// Arc tangent implementation.
- /// \param x first argument
- /// \param y second argument
- /// \return function value stored in single-preicision
- static expr atan2(float x, float y) { return expr(std::atan2(x, y)); }
+ /// Arc tangent implementation.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return function value stored in single-preicision
+ static expr atan2(float x, float y) { return expr(std::atan2(x, y)); }
- /// Hyperbolic sine implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr sinh(float arg) { return expr(std::sinh(arg)); }
+ /// Hyperbolic sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sinh(float arg) { return expr(std::sinh(arg)); }
- /// Hyperbolic cosine implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr cosh(float arg) { return expr(std::cosh(arg)); }
+ /// Hyperbolic cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cosh(float arg) { return expr(std::cosh(arg)); }
- /// Hyperbolic tangent implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr tanh(float arg) { return expr(std::tanh(arg)); }
+ /// Hyperbolic tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tanh(float arg) { return expr(std::tanh(arg)); }
- /// Hyperbolic area sine implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr asinh(float arg) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::asinh(arg));
-#else
- return expr(
- (arg == -std::numeric_limits<float>::infinity())
- ? arg
- : static_cast<float>(std::log(arg + std::sqrt(arg * arg + 1.0))));
-#endif
- }
+ /// Hyperbolic area sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr asinh(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::asinh(arg));
+ #else
+ return expr((arg==-std::numeric_limits<float>::infinity()) ? arg : static_cast<float>(std::log(arg+std::sqrt(arg*arg+1.0))));
+ #endif
+ }
- /// Hyperbolic area cosine implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr acosh(float arg) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::acosh(arg));
-#else
- return expr((arg < -1.0f) ? std::numeric_limits<float>::quiet_NaN()
- : static_cast<float>(std::log(
- arg + std::sqrt(arg * arg - 1.0))));
-#endif
- }
+ /// Hyperbolic area cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr acosh(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::acosh(arg));
+ #else
+ return expr((arg<-1.0f) ? std::numeric_limits<float>::quiet_NaN() : static_cast<float>(std::log(arg+std::sqrt(arg*arg-1.0))));
+ #endif
+ }
- /// Hyperbolic area tangent implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr atanh(float arg) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::atanh(arg));
-#else
- return expr(static_cast<float>(0.5 * std::log((1.0 + arg) / (1.0 - arg))));
-#endif
- }
+ /// Hyperbolic area tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr atanh(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::atanh(arg));
+ #else
+ return expr(static_cast<float>(0.5*std::log((1.0+arg)/(1.0-arg))));
+ #endif
+ }
- /// Error function implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr erf(float arg) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::erf(arg));
-#else
- return expr(static_cast<float>(erf(static_cast<double>(arg))));
-#endif
- }
+ /// Error function implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr erf(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::erf(arg));
+ #else
+ return expr(static_cast<float>(erf(static_cast<double>(arg))));
+ #endif
+ }
- /// Complementary implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr erfc(float arg) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::erfc(arg));
-#else
- return expr(static_cast<float>(1.0 - erf(static_cast<double>(arg))));
-#endif
- }
+ /// Complementary implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr erfc(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::erfc(arg));
+ #else
+ return expr(static_cast<float>(1.0-erf(static_cast<double>(arg))));
+ #endif
+ }
- /// Gamma logarithm implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr lgamma(float arg) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::lgamma(arg));
-#else
- if (builtin_isinf(arg)) return expr(std::numeric_limits<float>::infinity());
- if (arg < 0.0f) {
- float i, f = std::modf(-arg, &i);
- if (f == 0.0f) return expr(std::numeric_limits<float>::infinity());
- return expr(static_cast<float>(
- 1.1447298858494001741434273513531 -
- std::log(std::abs(std::sin(3.1415926535897932384626433832795 * f))) -
- lgamma(1.0 - arg)));
- }
- return expr(static_cast<float>(lgamma(static_cast<double>(arg))));
-#endif
- }
+ /// Gamma logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr lgamma(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::lgamma(arg));
+ #else
+ if(builtin_isinf(arg))
+ return expr(std::numeric_limits<float>::infinity());
+ if(arg < 0.0f)
+ {
+ float i, f = std::modf(-arg, &i);
+ if(f == 0.0f)
+ return expr(std::numeric_limits<float>::infinity());
+ return expr(static_cast<float>(1.1447298858494001741434273513531-
+ std::log(std::abs(std::sin(3.1415926535897932384626433832795*f)))-lgamma(1.0-arg)));
+ }
+ return expr(static_cast<float>(lgamma(static_cast<double>(arg))));
+ #endif
+ }
- /// Gamma implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr tgamma(float arg) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::tgamma(arg));
-#else
- if (arg == 0.0f)
- return builtin_signbit(arg)
- ? expr(-std::numeric_limits<float>::infinity())
- : expr(std::numeric_limits<float>::infinity());
- if (arg < 0.0f) {
- float i, f = std::modf(-arg, &i);
- if (f == 0.0f) return expr(std::numeric_limits<float>::quiet_NaN());
- double value = 3.1415926535897932384626433832795 /
- (std::sin(3.1415926535897932384626433832795 * f) *
- std::exp(lgamma(1.0 - arg)));
- return expr(
- static_cast<float>((std::fmod(i, 2.0f) == 0.0f) ? -value : value));
- }
- if (builtin_isinf(arg)) return expr(arg);
- return expr(static_cast<float>(std::exp(lgamma(static_cast<double>(arg)))));
-#endif
- }
+ /// Gamma implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tgamma(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::tgamma(arg));
+ #else
+ if(arg == 0.0f)
+ return builtin_signbit(arg) ? expr(-std::numeric_limits<float>::infinity()) : expr(std::numeric_limits<float>::infinity());
+ if(arg < 0.0f)
+ {
+ float i, f = std::modf(-arg, &i);
+ if(f == 0.0f)
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ double value = 3.1415926535897932384626433832795 / (std::sin(3.1415926535897932384626433832795*f)*std::exp(lgamma(1.0-arg)));
+ return expr(static_cast<float>((std::fmod(i, 2.0f)==0.0f) ? -value : value));
+ }
+ if(builtin_isinf(arg))
+ return expr(arg);
+ return expr(static_cast<float>(std::exp(lgamma(static_cast<double>(arg)))));
+ #endif
+ }
- /// Floor implementation.
- /// \param arg value to round
- /// \return rounded value
- static half floor(half arg) {
- return half(binary, round_half<std::round_toward_neg_infinity>(arg.data_));
- }
+ /// Floor implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half floor(half arg) { return half(binary, round_half<std::round_toward_neg_infinity>(arg.data_)); }
- /// Ceiling implementation.
- /// \param arg value to round
- /// \return rounded value
- static half ceil(half arg) {
- return half(binary, round_half<std::round_toward_infinity>(arg.data_));
- }
+ /// Ceiling implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half ceil(half arg) { return half(binary, round_half<std::round_toward_infinity>(arg.data_)); }
- /// Truncation implementation.
- /// \param arg value to round
- /// \return rounded value
- static half trunc(half arg) {
- return half(binary, round_half<std::round_toward_zero>(arg.data_));
- }
+ /// Truncation implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half trunc(half arg) { return half(binary, round_half<std::round_toward_zero>(arg.data_)); }
- /// Nearest integer implementation.
- /// \param arg value to round
- /// \return rounded value
- static half round(half arg) { return half(binary, round_half_up(arg.data_)); }
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half round(half arg) { return half(binary, round_half_up(arg.data_)); }
- /// Nearest integer implementation.
- /// \param arg value to round
- /// \return rounded value
- static long lround(half arg) { return detail::half2int_up<long>(arg.data_); }
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long lround(half arg) { return detail::half2int_up<long>(arg.data_); }
- /// Nearest integer implementation.
- /// \param arg value to round
- /// \return rounded value
- static half rint(half arg) {
- return half(binary, round_half<half::round_style>(arg.data_));
- }
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half rint(half arg) { return half(binary, round_half<half::round_style>(arg.data_)); }
- /// Nearest integer implementation.
- /// \param arg value to round
- /// \return rounded value
- static long lrint(half arg) {
- return detail::half2int<half::round_style, long>(arg.data_);
- }
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long lrint(half arg) { return detail::half2int<half::round_style,long>(arg.data_); }
-#if HALF_ENABLE_CPP11_LONG_LONG
- /// Nearest integer implementation.
- /// \param arg value to round
- /// \return rounded value
- static long long llround(half arg) {
- return detail::half2int_up<long long>(arg.data_);
- }
+ #if HALF_ENABLE_CPP11_LONG_LONG
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long long llround(half arg) { return detail::half2int_up<long long>(arg.data_); }
- /// Nearest integer implementation.
- /// \param arg value to round
- /// \return rounded value
- static long long llrint(half arg) {
- return detail::half2int<half::round_style, long long>(arg.data_);
- }
-#endif
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long long llrint(half arg) { return detail::half2int<half::round_style,long long>(arg.data_); }
+ #endif
- /// Decompression implementation.
- /// \param arg number to decompress
- /// \param exp address to store exponent at
- /// \return normalized significant
- static half frexp(half arg, int *exp) {
- int m = arg.data_ & 0x7FFF, e = -14;
- if (m >= 0x7C00 || !m) return *exp = 0, arg;
- for (; m < 0x400; m <<= 1, --e)
- ;
- return *exp = e + (m >> 10),
- half(binary, (arg.data_ & 0x8000) | 0x3800 | (m & 0x3FF));
- }
+ /// Decompression implementation.
+ /// \param arg number to decompress
+ /// \param exp address to store exponent at
+ /// \return normalized significant
+ static half frexp(half arg, int *exp)
+ {
+ int m = arg.data_ & 0x7FFF, e = -14;
+ if(m >= 0x7C00 || !m)
+ return *exp = 0, arg;
+ for(; m<0x400; m<<=1,--e) ;
+ return *exp = e+(m>>10), half(binary, (arg.data_&0x8000)|0x3800|(m&0x3FF));
+ }
- /// Decompression implementation.
- /// \param arg number to decompress
- /// \param iptr address to store integer part at
- /// \return fractional part
- static half modf(half arg, half *iptr) {
- unsigned int e = arg.data_ & 0x7FFF;
- if (e >= 0x6400)
- return *iptr = arg, half(binary, arg.data_ & (0x8000U | -(e > 0x7C00)));
- if (e < 0x3C00) return iptr->data_ = arg.data_ & 0x8000, arg;
- e >>= 10;
- unsigned int mask = (1 << (25 - e)) - 1, m = arg.data_ & mask;
- iptr->data_ = arg.data_ & ~mask;
- if (!m) return half(binary, arg.data_ & 0x8000);
- for (; m < 0x400; m <<= 1, --e)
- ;
- return half(binary, static_cast<uint16>((arg.data_ & 0x8000) | (e << 10) |
- (m & 0x3FF)));
- }
+ /// Decompression implementation.
+ /// \param arg number to decompress
+ /// \param iptr address to store integer part at
+ /// \return fractional part
+ static half modf(half arg, half *iptr)
+ {
+ unsigned int e = arg.data_ & 0x7FFF;
+ if(e >= 0x6400)
+ return *iptr = arg, half(binary, arg.data_&(0x8000U|-(e>0x7C00)));
+ if(e < 0x3C00)
+ return iptr->data_ = arg.data_ & 0x8000, arg;
+ e >>= 10;
+ unsigned int mask = (1<<(25-e)) - 1, m = arg.data_ & mask;
+ iptr->data_ = arg.data_ & ~mask;
+ if(!m)
+ return half(binary, arg.data_&0x8000);
+ for(; m<0x400; m<<=1,--e) ;
+ return half(binary, static_cast<uint16>((arg.data_&0x8000)|(e<<10)|(m&0x3FF)));
+ }
- /// Scaling implementation.
- /// \param arg number to scale
- /// \param exp power of two to scale by
- /// \return scaled number
- static half scalbln(half arg, long exp) {
- unsigned int m = arg.data_ & 0x7FFF;
- if (m >= 0x7C00 || !m) return arg;
- for (; m < 0x400; m <<= 1, --exp)
- ;
- exp += m >> 10;
- uint16 value = arg.data_ & 0x8000;
- if (exp > 30) {
- if (half::round_style == std::round_toward_zero)
- value |= 0x7BFF;
- else if (half::round_style == std::round_toward_infinity)
- value |= 0x7C00 - (value >> 15);
- else if (half::round_style == std::round_toward_neg_infinity)
- value |= 0x7BFF + (value >> 15);
- else
- value |= 0x7C00;
- } else if (exp > 0)
- value |= (exp << 10) | (m & 0x3FF);
- else if (exp > -11) {
- m = (m & 0x3FF) | 0x400;
- if (half::round_style == std::round_to_nearest) {
- m += 1 << -exp;
-#if HALF_ROUND_TIES_TO_EVEN
- m -= (m >> (1 - exp)) & 1;
-#endif
- } else if (half::round_style == std::round_toward_infinity)
- m += ((value >> 15) - 1) & ((1 << (1 - exp)) - 1U);
- else if (half::round_style == std::round_toward_neg_infinity)
- m += -(value >> 15) & ((1 << (1 - exp)) - 1U);
- value |= m >> (1 - exp);
- } else if (half::round_style == std::round_toward_infinity)
- value -= (value >> 15) - 1;
- else if (half::round_style == std::round_toward_neg_infinity)
- value += value >> 15;
- return half(binary, value);
- }
+ /// Scaling implementation.
+ /// \param arg number to scale
+ /// \param exp power of two to scale by
+ /// \return scaled number
+ static half scalbln(half arg, long exp)
+ {
+ unsigned int m = arg.data_ & 0x7FFF;
+ if(m >= 0x7C00 || !m)
+ return arg;
+ for(; m<0x400; m<<=1,--exp) ;
+ exp += m >> 10;
+ uint16 value = arg.data_ & 0x8000;
+ if(exp > 30)
+ {
+ if(half::round_style == std::round_toward_zero)
+ value |= 0x7BFF;
+ else if(half::round_style == std::round_toward_infinity)
+ value |= 0x7C00 - (value>>15);
+ else if(half::round_style == std::round_toward_neg_infinity)
+ value |= 0x7BFF + (value>>15);
+ else
+ value |= 0x7C00;
+ }
+ else if(exp > 0)
+ value |= (exp<<10) | (m&0x3FF);
+ else if(exp > -11)
+ {
+ m = (m&0x3FF) | 0x400;
+ if(half::round_style == std::round_to_nearest)
+ {
+ m += 1 << -exp;
+ #if HALF_ROUND_TIES_TO_EVEN
+ m -= (m>>(1-exp)) & 1;
+ #endif
+ }
+ else if(half::round_style == std::round_toward_infinity)
+ m += ((value>>15)-1) & ((1<<(1-exp))-1U);
+ else if(half::round_style == std::round_toward_neg_infinity)
+ m += -(value>>15) & ((1<<(1-exp))-1U);
+ value |= m >> (1-exp);
+ }
+ else if(half::round_style == std::round_toward_infinity)
+ value -= (value>>15) - 1;
+ else if(half::round_style == std::round_toward_neg_infinity)
+ value += value >> 15;
+ return half(binary, value);
+ }
+
+ /// Exponent implementation.
+ /// \param arg number to query
+ /// \return floating point exponent
+ static int ilogb(half arg)
+ {
+ int abs = arg.data_ & 0x7FFF;
+ if(!abs)
+ return FP_ILOGB0;
+ if(abs < 0x7C00)
+ {
+ int exp = (abs>>10) - 15;
+ if(abs < 0x400)
+ for(; abs<0x200; abs<<=1,--exp) ;
+ return exp;
+ }
+ if(abs > 0x7C00)
+ return FP_ILOGBNAN;
+ return INT_MAX;
+ }
- /// Exponent implementation.
- /// \param arg number to query
- /// \return floating point exponent
- static int ilogb(half arg) {
- int abs = arg.data_ & 0x7FFF;
- if (!abs) return FP_ILOGB0;
- if (abs < 0x7C00) {
- int exp = (abs >> 10) - 15;
- if (abs < 0x400)
- for (; abs < 0x200; abs <<= 1, --exp)
- ;
- return exp;
- }
- if (abs > 0x7C00) return FP_ILOGBNAN;
- return INT_MAX;
- }
+ /// Exponent implementation.
+ /// \param arg number to query
+ /// \return floating point exponent
+ static half logb(half arg)
+ {
+ int abs = arg.data_ & 0x7FFF;
+ if(!abs)
+ return half(binary, 0xFC00);
+ if(abs < 0x7C00)
+ {
+ int exp = (abs>>10) - 15;
+ if(abs < 0x400)
+ for(; abs<0x200; abs<<=1,--exp) ;
+ uint16 bits = (exp<0) << 15;
+ if(exp)
+ {
+ unsigned int m = std::abs(exp) << 6, e = 18;
+ for(; m<0x400; m<<=1,--e) ;
+ bits |= (e<<10) + m;
+ }
+ return half(binary, bits);
+ }
+ if(abs > 0x7C00)
+ return arg;
+ return half(binary, 0x7C00);
+ }
- /// Exponent implementation.
- /// \param arg number to query
- /// \return floating point exponent
- static half logb(half arg) {
- int abs = arg.data_ & 0x7FFF;
- if (!abs) return half(binary, 0xFC00);
- if (abs < 0x7C00) {
- int exp = (abs >> 10) - 15;
- if (abs < 0x400)
- for (; abs < 0x200; abs <<= 1, --exp)
- ;
- uint16 bits = (exp < 0) << 15;
- if (exp) {
- unsigned int m = std::abs(exp) << 6, e = 18;
- for (; m < 0x400; m <<= 1, --e)
- ;
- bits |= (e << 10) + m;
- }
- return half(binary, bits);
- }
- if (abs > 0x7C00) return arg;
- return half(binary, 0x7C00);
- }
+ /// Enumeration implementation.
+ /// \param from number to increase/decrease
+ /// \param to direction to enumerate into
+ /// \return next representable number
+ static half nextafter(half from, half to)
+ {
+ uint16 fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF;
+ if(fabs > 0x7C00)
+ return from;
+ if(tabs > 0x7C00 || from.data_ == to.data_ || !(fabs|tabs))
+ return to;
+ if(!fabs)
+ return half(binary, (to.data_&0x8000)+1);
+ bool lt = ((fabs==from.data_) ? static_cast<int>(fabs) : -static_cast<int>(fabs)) <
+ ((tabs==to.data_) ? static_cast<int>(tabs) : -static_cast<int>(tabs));
+ return half(binary, from.data_+(((from.data_>>15)^static_cast<unsigned>(lt))<<1)-1);
+ }
- /// Enumeration implementation.
- /// \param from number to increase/decrease
- /// \param to direction to enumerate into
- /// \return next representable number
- static half nextafter(half from, half to) {
- uint16 fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF;
- if (fabs > 0x7C00) return from;
- if (tabs > 0x7C00 || from.data_ == to.data_ || !(fabs | tabs)) return to;
- if (!fabs) return half(binary, (to.data_ & 0x8000) + 1);
- bool lt =
- ((fabs == from.data_) ? static_cast<int>(fabs)
- : -static_cast<int>(fabs)) <
- ((tabs == to.data_) ? static_cast<int>(tabs) : -static_cast<int>(tabs));
- return half(binary,
- from.data_ +
- (((from.data_ >> 15) ^ static_cast<unsigned>(lt)) << 1) -
- 1);
- }
+ /// Enumeration implementation.
+ /// \param from number to increase/decrease
+ /// \param to direction to enumerate into
+ /// \return next representable number
+ static half nexttoward(half from, long double to)
+ {
+ if(isnan(from))
+ return from;
+ long double lfrom = static_cast<long double>(from);
+ if(builtin_isnan(to) || lfrom == to)
+ return half(static_cast<float>(to));
+ if(!(from.data_&0x7FFF))
+ return half(binary, (static_cast<detail::uint16>(builtin_signbit(to))<<15)+1);
+ return half(binary, from.data_+(((from.data_>>15)^static_cast<unsigned>(lfrom<to))<<1)-1);
+ }
- /// Enumeration implementation.
- /// \param from number to increase/decrease
- /// \param to direction to enumerate into
- /// \return next representable number
- static half nexttoward(half from, long double to) {
- if (isnan(from)) return from;
- long double lfrom = static_cast<long double>(from);
- if (builtin_isnan(to) || lfrom == to) return half(static_cast<float>(to));
- if (!(from.data_ & 0x7FFF))
- return half(binary,
- (static_cast<detail::uint16>(builtin_signbit(to)) << 15) + 1);
- return half(
- binary,
- from.data_ +
- (((from.data_ >> 15) ^ static_cast<unsigned>(lfrom < to)) << 1) -
- 1);
- }
+ /// Sign implementation
+ /// \param x first operand
+ /// \param y second operand
+ /// \return composed value
+ static half copysign(half x, half y) { return half(binary, x.data_^((x.data_^y.data_)&0x8000)); }
- /// Sign implementation
- /// \param x first operand
- /// \param y second operand
- /// \return composed value
- static half copysign(half x, half y) {
- return half(binary, x.data_ ^ ((x.data_ ^ y.data_) & 0x8000));
- }
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if infinite number
+ /// \retval false else
+ static int fpclassify(half arg)
+ {
+ unsigned int abs = arg.data_ & 0x7FFF;
+ return abs ? ((abs>0x3FF) ? ((abs>=0x7C00) ? ((abs>0x7C00) ? FP_NAN : FP_INFINITE) : FP_NORMAL) :FP_SUBNORMAL) : FP_ZERO;
+ }
- /// Classification implementation.
- /// \param arg value to classify
- /// \retval true if infinite number
- /// \retval false else
- static int fpclassify(half arg) {
- unsigned int abs = arg.data_ & 0x7FFF;
- return abs ? ((abs > 0x3FF) ? ((abs >= 0x7C00)
- ? ((abs > 0x7C00) ? FP_NAN : FP_INFINITE)
- : FP_NORMAL)
- : FP_SUBNORMAL)
- : FP_ZERO;
- }
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if finite number
+ /// \retval false else
+ static bool isfinite(half arg) { return (arg.data_&0x7C00) != 0x7C00; }
- /// Classification implementation.
- /// \param arg value to classify
- /// \retval true if finite number
- /// \retval false else
- static bool isfinite(half arg) { return (arg.data_ & 0x7C00) != 0x7C00; }
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if infinite number
+ /// \retval false else
+ static bool isinf(half arg) { return (arg.data_&0x7FFF) == 0x7C00; }
- /// Classification implementation.
- /// \param arg value to classify
- /// \retval true if infinite number
- /// \retval false else
- static bool isinf(half arg) { return (arg.data_ & 0x7FFF) == 0x7C00; }
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if not a number
+ /// \retval false else
+ static bool isnan(half arg) { return (arg.data_&0x7FFF) > 0x7C00; }
- /// Classification implementation.
- /// \param arg value to classify
- /// \retval true if not a number
- /// \retval false else
- static bool isnan(half arg) { return (arg.data_ & 0x7FFF) > 0x7C00; }
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if normal number
+ /// \retval false else
+ static bool isnormal(half arg) { return ((arg.data_&0x7C00)!=0) & ((arg.data_&0x7C00)!=0x7C00); }
- /// Classification implementation.
- /// \param arg value to classify
- /// \retval true if normal number
- /// \retval false else
- static bool isnormal(half arg) {
- return ((arg.data_ & 0x7C00) != 0) & ((arg.data_ & 0x7C00) != 0x7C00);
- }
+ /// Sign bit implementation.
+ /// \param arg value to check
+ /// \retval true if signed
+ /// \retval false if unsigned
+ static bool signbit(half arg) { return (arg.data_&0x8000) != 0; }
- /// Sign bit implementation.
- /// \param arg value to check
- /// \retval true if signed
- /// \retval false if unsigned
- static bool signbit(half arg) { return (arg.data_ & 0x8000) != 0; }
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands equal
+ /// \retval false else
+ static bool isequal(half x, half y) { return (x.data_==y.data_ || !((x.data_|y.data_)&0x7FFF)) && !isnan(x); }
- /// Comparison implementation.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if operands equal
- /// \retval false else
- static bool isequal(half x, half y) {
- return (x.data_ == y.data_ || !((x.data_ | y.data_) & 0x7FFF)) && !isnan(x);
- }
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands not equal
+ /// \retval false else
+ static bool isnotequal(half x, half y) { return (x.data_!=y.data_ && ((x.data_|y.data_)&0x7FFF)) || isnan(x); }
- /// Comparison implementation.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if operands not equal
- /// \retval false else
- static bool isnotequal(half x, half y) {
- return (x.data_ != y.data_ && ((x.data_ | y.data_) & 0x7FFF)) || isnan(x);
- }
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x > \a y
+ /// \retval false else
+ static bool isgreater(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs));
+ }
- /// Comparison implementation.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if \a x > \a y
- /// \retval false else
- static bool isgreater(half x, half y) {
- int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
- return xabs <= 0x7C00 && yabs <= 0x7C00 &&
- (((xabs == x.data_) ? xabs : -xabs) >
- ((yabs == y.data_) ? yabs : -yabs));
- }
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x >= \a y
+ /// \retval false else
+ static bool isgreaterequal(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) >= ((yabs==y.data_) ? yabs : -yabs));
+ }
- /// Comparison implementation.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if \a x >= \a y
- /// \retval false else
- static bool isgreaterequal(half x, half y) {
- int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
- return xabs <= 0x7C00 && yabs <= 0x7C00 &&
- (((xabs == x.data_) ? xabs : -xabs) >=
- ((yabs == y.data_) ? yabs : -yabs));
- }
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x < \a y
+ /// \retval false else
+ static bool isless(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs));
+ }
- /// Comparison implementation.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if \a x < \a y
- /// \retval false else
- static bool isless(half x, half y) {
- int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
- return xabs <= 0x7C00 && yabs <= 0x7C00 &&
- (((xabs == x.data_) ? xabs : -xabs) <
- ((yabs == y.data_) ? yabs : -yabs));
- }
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x <= \a y
+ /// \retval false else
+ static bool islessequal(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) <= ((yabs==y.data_) ? yabs : -yabs));
+ }
- /// Comparison implementation.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if \a x <= \a y
- /// \retval false else
- static bool islessequal(half x, half y) {
- int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
- return xabs <= 0x7C00 && yabs <= 0x7C00 &&
- (((xabs == x.data_) ? xabs : -xabs) <=
- ((yabs == y.data_) ? yabs : -yabs));
- }
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if either \a x > \a y nor \a x < \a y
+ /// \retval false else
+ static bool islessgreater(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if(xabs > 0x7C00 || yabs > 0x7C00)
+ return false;
+ int a = (xabs==x.data_) ? xabs : -xabs, b = (yabs==y.data_) ? yabs : -yabs;
+ return a < b || a > b;
+ }
- /// Comparison implementation.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if either \a x > \a y nor \a x < \a y
- /// \retval false else
- static bool islessgreater(half x, half y) {
- int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
- if (xabs > 0x7C00 || yabs > 0x7C00) return false;
- int a = (xabs == x.data_) ? xabs : -xabs,
- b = (yabs == y.data_) ? yabs : -yabs;
- return a < b || a > b;
- }
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operand unordered
+ /// \retval false else
+ static bool isunordered(half x, half y) { return isnan(x) || isnan(y); }
- /// Comparison implementation.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if operand unordered
- /// \retval false else
- static bool isunordered(half x, half y) { return isnan(x) || isnan(y); }
+ private:
+ static double erf(double arg)
+ {
+ if(builtin_isinf(arg))
+ return (arg<0.0) ? -1.0 : 1.0;
+ double x2 = arg * arg, ax2 = 0.147 * x2, value = std::sqrt(1.0-std::exp(-x2*(1.2732395447351626861510701069801+ax2)/(1.0+ax2)));
+ return builtin_signbit(arg) ? -value : value;
+ }
- private:
- static double erf(double arg) {
- if (builtin_isinf(arg)) return (arg < 0.0) ? -1.0 : 1.0;
- double x2 = arg * arg, ax2 = 0.147 * x2,
- value = std::sqrt(
- 1.0 - std::exp(-x2 * (1.2732395447351626861510701069801 + ax2) /
- (1.0 + ax2)));
- return builtin_signbit(arg) ? -value : value;
- }
+ static double lgamma(double arg)
+ {
+ double v = 1.0;
+ for(; arg<8.0; ++arg) v *= arg;
+ double w = 1.0 / (arg*arg);
+ return (((((((-0.02955065359477124183006535947712*w+0.00641025641025641025641025641026)*w+
+ -0.00191752691752691752691752691753)*w+8.4175084175084175084175084175084e-4)*w+
+ -5.952380952380952380952380952381e-4)*w+7.9365079365079365079365079365079e-4)*w+
+ -0.00277777777777777777777777777778)*w+0.08333333333333333333333333333333)/arg +
+ 0.91893853320467274178032973640562 - std::log(v) - arg + (arg-0.5) * std::log(arg);
+ }
+ };
- static double lgamma(double arg) {
- double v = 1.0;
- for (; arg < 8.0; ++arg) v *= arg;
- double w = 1.0 / (arg * arg);
- return (((((((-0.02955065359477124183006535947712 * w +
- 0.00641025641025641025641025641026) *
- w +
- -0.00191752691752691752691752691753) *
- w +
- 8.4175084175084175084175084175084e-4) *
- w +
- -5.952380952380952380952380952381e-4) *
- w +
- 7.9365079365079365079365079365079e-4) *
- w +
- -0.00277777777777777777777777777778) *
- w +
- 0.08333333333333333333333333333333) /
- arg +
- 0.91893853320467274178032973640562 - std::log(v) - arg +
- (arg - 0.5) * std::log(arg);
- }
-};
+ /// Wrapper for unary half-precision functions needing specialization for individual argument types.
+ /// \tparam T argument type
+ template<typename T> struct unary_specialized
+ {
+ /// Negation implementation.
+ /// \param arg value to negate
+ /// \return negated value
+ static HALF_CONSTEXPR half negate(half arg) { return half(binary, arg.data_^0x8000); }
-/// Wrapper for unary half-precision functions needing specialization for
-/// individual argument types.
-/// \tparam T argument type
-template <typename T>
-struct unary_specialized {
- /// Negation implementation.
- /// \param arg value to negate
- /// \return negated value
- static HALF_CONSTEXPR half negate(half arg) {
- return half(binary, arg.data_ ^ 0x8000);
- }
+ /// Absolute value implementation.
+ /// \param arg function argument
+ /// \return absolute value
+ static half fabs(half arg) { return half(binary, arg.data_&0x7FFF); }
+ };
+ template<> struct unary_specialized<expr>
+ {
+ static HALF_CONSTEXPR expr negate(float arg) { return expr(-arg); }
+ static expr fabs(float arg) { return expr(std::fabs(arg)); }
+ };
- /// Absolute value implementation.
- /// \param arg function argument
- /// \return absolute value
- static half fabs(half arg) { return half(binary, arg.data_ & 0x7FFF); }
-};
-template <>
-struct unary_specialized<expr> {
- static HALF_CONSTEXPR expr negate(float arg) { return expr(-arg); }
- static expr fabs(float arg) { return expr(std::fabs(arg)); }
-};
+ /// Wrapper for binary half-precision functions needing specialization for individual argument types.
+ /// \tparam T first argument type
+ /// \tparam U first argument type
+ template<typename T,typename U> struct binary_specialized
+ {
+ /// Minimum implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return minimum value
+ static expr fmin(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fmin(x, y));
+ #else
+ if(builtin_isnan(x))
+ return expr(y);
+ if(builtin_isnan(y))
+ return expr(x);
+ return expr(std::min(x, y));
+ #endif
+ }
-/// Wrapper for binary half-precision functions needing specialization for
-/// individual argument types.
-/// \tparam T first argument type
-/// \tparam U first argument type
-template <typename T, typename U>
-struct binary_specialized {
- /// Minimum implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return minimum value
- static expr fmin(float x, float y) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::fmin(x, y));
-#else
- if (builtin_isnan(x)) return expr(y);
- if (builtin_isnan(y)) return expr(x);
- return expr(std::min(x, y));
-#endif
- }
+ /// Maximum implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return maximum value
+ static expr fmax(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fmax(x, y));
+ #else
+ if(builtin_isnan(x))
+ return expr(y);
+ if(builtin_isnan(y))
+ return expr(x);
+ return expr(std::max(x, y));
+ #endif
+ }
+ };
+ template<> struct binary_specialized<half,half>
+ {
+ static half fmin(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if(xabs > 0x7C00)
+ return y;
+ if(yabs > 0x7C00)
+ return x;
+ return (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs)) ? y : x;
+ }
+ static half fmax(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if(xabs > 0x7C00)
+ return y;
+ if(yabs > 0x7C00)
+ return x;
+ return (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs)) ? y : x;
+ }
+ };
- /// Maximum implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return maximum value
- static expr fmax(float x, float y) {
-#if HALF_ENABLE_CPP11_CMATH
- return expr(std::fmax(x, y));
-#else
- if (builtin_isnan(x)) return expr(y);
- if (builtin_isnan(y)) return expr(x);
- return expr(std::max(x, y));
-#endif
- }
-};
-template <>
-struct binary_specialized<half, half> {
- static half fmin(half x, half y) {
- int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
- if (xabs > 0x7C00) return y;
- if (yabs > 0x7C00) return x;
- return (((xabs == x.data_) ? xabs : -xabs) >
- ((yabs == y.data_) ? yabs : -yabs))
- ? y
- : x;
- }
- static half fmax(half x, half y) {
- int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
- if (xabs > 0x7C00) return y;
- if (yabs > 0x7C00) return x;
- return (((xabs == x.data_) ? xabs : -xabs) <
- ((yabs == y.data_) ? yabs : -yabs))
- ? y
- : x;
- }
-};
+ /// Helper class for half casts.
+ /// This class template has to be specialized for all valid cast argument to define an appropriate static `cast` member
+ /// function and a corresponding `type` member denoting its return type.
+ /// \tparam T destination type
+ /// \tparam U source type
+ /// \tparam R rounding mode to use
+ template<typename T,typename U,std::float_round_style R=(std::float_round_style)(HALF_ROUND_STYLE)> struct half_caster {};
+ template<typename U,std::float_round_style R> struct half_caster<half,U,R>
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<U>::value, "half_cast from non-arithmetic type unsupported");
+ #endif
-/// Helper class for half casts.
-/// This class template has to be specialized for all valid cast argument to
-/// define an appropriate static `cast` member
-/// function and a corresponding `type` member denoting its return type.
-/// \tparam T destination type
-/// \tparam U source type
-/// \tparam R rounding mode to use
-template <typename T, typename U,
- std::float_round_style R = (std::float_round_style)(HALF_ROUND_STYLE)>
-struct half_caster {};
-template <typename U, std::float_round_style R>
-struct half_caster<half, U, R> {
-#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
- static_assert(std::is_arithmetic<U>::value,
- "half_cast from non-arithmetic type unsupported");
-#endif
+ static half cast(U arg) { return cast_impl(arg, is_float<U>()); };
- static half cast(U arg) { return cast_impl(arg, is_float<U>()); };
+ private:
+ static half cast_impl(U arg, true_type) { return half(binary, float2half<R>(arg)); }
+ static half cast_impl(U arg, false_type) { return half(binary, int2half<R>(arg)); }
+ };
+ template<typename T,std::float_round_style R> struct half_caster<T,half,R>
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<T>::value, "half_cast to non-arithmetic type unsupported");
+ #endif
- private:
- static half cast_impl(U arg, true_type) {
- return half(binary, float2half<R>(arg));
- }
- static half cast_impl(U arg, false_type) {
- return half(binary, int2half<R>(arg));
- }
-};
-template <typename T, std::float_round_style R>
-struct half_caster<T, half, R> {
-#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
- static_assert(std::is_arithmetic<T>::value,
- "half_cast to non-arithmetic type unsupported");
-#endif
+ static T cast(half arg) { return cast_impl(arg, is_float<T>()); }
- static T cast(half arg) { return cast_impl(arg, is_float<T>()); }
+ private:
+ static T cast_impl(half arg, true_type) { return half2float<T>(arg.data_); }
+ static T cast_impl(half arg, false_type) { return half2int<R,T>(arg.data_); }
+ };
+ template<typename T,std::float_round_style R> struct half_caster<T,expr,R>
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<T>::value, "half_cast to non-arithmetic type unsupported");
+ #endif
- private:
- static T cast_impl(half arg, true_type) { return half2float<T>(arg.data_); }
- static T cast_impl(half arg, false_type) { return half2int<R, T>(arg.data_); }
-};
-template <typename T, std::float_round_style R>
-struct half_caster<T, expr, R> {
-#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
- static_assert(std::is_arithmetic<T>::value,
- "half_cast to non-arithmetic type unsupported");
-#endif
+ static T cast(expr arg) { return cast_impl(arg, is_float<T>()); }
- static T cast(expr arg) { return cast_impl(arg, is_float<T>()); }
+ private:
+ static T cast_impl(float arg, true_type) { return static_cast<T>(arg); }
+ static T cast_impl(half arg, false_type) { return half2int<R,T>(arg.data_); }
+ };
+ template<std::float_round_style R> struct half_caster<half,half,R>
+ {
+ static half cast(half arg) { return arg; }
+ };
+ template<std::float_round_style R> struct half_caster<half,expr,R> : half_caster<half,half,R> {};
- private:
- static T cast_impl(float arg, true_type) { return static_cast<T>(arg); }
- static T cast_impl(half arg, false_type) { return half2int<R, T>(arg.data_); }
-};
-template <std::float_round_style R>
-struct half_caster<half, half, R> {
- static half cast(half arg) { return arg; }
-};
-template <std::float_round_style R>
-struct half_caster<half, expr, R> : half_caster<half, half, R> {};
+ /// \name Comparison operators
+ /// \{
-/// \name Comparison operators
-/// \{
+ /// Comparison for equality.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands equal
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator==(T x, U y) { return functions::isequal(x, y); }
-/// Comparison for equality.
-/// \param x first operand
-/// \param y second operand
-/// \retval true if operands equal
-/// \retval false else
-template <typename T, typename U>
-typename enable<bool, T, U>::type operator==(T x, U y) {
- return functions::isequal(x, y);
-}
+ /// Comparison for inequality.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands not equal
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator!=(T x, U y) { return functions::isnotequal(x, y); }
-/// Comparison for inequality.
-/// \param x first operand
-/// \param y second operand
-/// \retval true if operands not equal
-/// \retval false else
-template <typename T, typename U>
-typename enable<bool, T, U>::type operator!=(T x, U y) {
- return functions::isnotequal(x, y);
-}
+ /// Comparison for less than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less than \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator<(T x, U y) { return functions::isless(x, y); }
-/// Comparison for less than.
-/// \param x first operand
-/// \param y second operand
-/// \retval true if \a x less than \a y
-/// \retval false else
-template <typename T, typename U>
-typename enable<bool, T, U>::type operator<(T x, U y) {
- return functions::isless(x, y);
-}
+ /// Comparison for greater than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater than \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator>(T x, U y) { return functions::isgreater(x, y); }
-/// Comparison for greater than.
-/// \param x first operand
-/// \param y second operand
-/// \retval true if \a x greater than \a y
-/// \retval false else
-template <typename T, typename U>
-typename enable<bool, T, U>::type operator>(T x, U y) {
- return functions::isgreater(x, y);
-}
+ /// Comparison for less equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less equal \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator<=(T x, U y) { return functions::islessequal(x, y); }
-/// Comparison for less equal.
-/// \param x first operand
-/// \param y second operand
-/// \retval true if \a x less equal \a y
-/// \retval false else
-template <typename T, typename U>
-typename enable<bool, T, U>::type operator<=(T x, U y) {
- return functions::islessequal(x, y);
-}
+ /// Comparison for greater equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater equal \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator>=(T x, U y) { return functions::isgreaterequal(x, y); }
-/// Comparison for greater equal.
-/// \param x first operand
-/// \param y second operand
-/// \retval true if \a x greater equal \a y
-/// \retval false else
-template <typename T, typename U>
-typename enable<bool, T, U>::type operator>=(T x, U y) {
- return functions::isgreaterequal(x, y);
-}
+ /// \}
+ /// \name Arithmetic operators
+ /// \{
-/// \}
-/// \name Arithmetic operators
-/// \{
+ /// Add halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return sum of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator+(T x, U y) { return functions::plus(x, y); }
-/// Add halfs.
-/// \param x left operand
-/// \param y right operand
-/// \return sum of half expressions
-template <typename T, typename U>
-typename enable<expr, T, U>::type operator+(T x, U y) {
- return functions::plus(x, y);
-}
+ /// Subtract halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return difference of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator-(T x, U y) { return functions::minus(x, y); }
-/// Subtract halfs.
-/// \param x left operand
-/// \param y right operand
-/// \return difference of half expressions
-template <typename T, typename U>
-typename enable<expr, T, U>::type operator-(T x, U y) {
- return functions::minus(x, y);
-}
+ /// Multiply halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return product of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator*(T x, U y) { return functions::multiplies(x, y); }
-/// Multiply halfs.
-/// \param x left operand
-/// \param y right operand
-/// \return product of half expressions
-template <typename T, typename U>
-typename enable<expr, T, U>::type operator*(T x, U y) {
- return functions::multiplies(x, y);
-}
+ /// Divide halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return quotient of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator/(T x, U y) { return functions::divides(x, y); }
-/// Divide halfs.
-/// \param x left operand
-/// \param y right operand
-/// \return quotient of half expressions
-template <typename T, typename U>
-typename enable<expr, T, U>::type operator/(T x, U y) {
- return functions::divides(x, y);
-}
+ /// Identity.
+ /// \param arg operand
+ /// \return uncahnged operand
+ template<typename T> HALF_CONSTEXPR typename enable<T,T>::type operator+(T arg) { return arg; }
-/// Identity.
-/// \param arg operand
-/// \return uncahnged operand
-template <typename T>
-HALF_CONSTEXPR typename enable<T, T>::type operator+(T arg) {
- return arg;
-}
+ /// Negation.
+ /// \param arg operand
+ /// \return negated operand
+ template<typename T> HALF_CONSTEXPR typename enable<T,T>::type operator-(T arg) { return unary_specialized<T>::negate(arg); }
-/// Negation.
-/// \param arg operand
-/// \return negated operand
-template <typename T>
-HALF_CONSTEXPR typename enable<T, T>::type operator-(T arg) {
- return unary_specialized<T>::negate(arg);
-}
+ /// \}
+ /// \name Input and output
+ /// \{
-/// \}
-/// \name Input and output
-/// \{
+ /// Output operator.
+ /// \param out output stream to write into
+ /// \param arg half expression to write
+ /// \return reference to output stream
+ template<typename T,typename charT,typename traits> typename enable<std::basic_ostream<charT,traits>&,T>::type
+ operator<<(std::basic_ostream<charT,traits> &out, T arg) { return functions::write(out, arg); }
-/// Output operator.
-/// \param out output stream to write into
-/// \param arg half expression to write
-/// \return reference to output stream
-template <typename T, typename charT, typename traits>
-typename enable<std::basic_ostream<charT, traits> &, T>::type operator<<(
- std::basic_ostream<charT, traits> &out, T arg) {
- return functions::write(out, arg);
-}
+ /// Input operator.
+ /// \param in input stream to read from
+ /// \param arg half to read into
+ /// \return reference to input stream
+ template<typename charT,typename traits> std::basic_istream<charT,traits>&
+ operator>>(std::basic_istream<charT,traits> &in, half &arg) { return functions::read(in, arg); }
-/// Input operator.
-/// \param in input stream to read from
-/// \param arg half to read into
-/// \return reference to input stream
-template <typename charT, typename traits>
-std::basic_istream<charT, traits> &operator>>(
- std::basic_istream<charT, traits> &in, half &arg) {
- return functions::read(in, arg);
-}
+ /// \}
+ /// \name Basic mathematical operations
+ /// \{
-/// \}
-/// \name Basic mathematical operations
-/// \{
+ /// Absolute value.
+ /// \param arg operand
+ /// \return absolute value of \a arg
+// template<typename T> typename enable<T,T>::type abs(T arg) { return unary_specialized<T>::fabs(arg); }
+ inline half abs(half arg) { return unary_specialized<half>::fabs(arg); }
+ inline expr abs(expr arg) { return unary_specialized<expr>::fabs(arg); }
-/// Absolute value.
-/// \param arg operand
-/// \return absolute value of \a arg
-// template<typename T> typename enable<T,T>::type abs(T arg) {
-//return unary_specialized<T>::fabs(arg); }
-inline half abs(half arg) { return unary_specialized<half>::fabs(arg); }
-inline expr abs(expr arg) { return unary_specialized<expr>::fabs(arg); }
+ /// Absolute value.
+ /// \param arg operand
+ /// \return absolute value of \a arg
+// template<typename T> typename enable<T,T>::type fabs(T arg) { return unary_specialized<T>::fabs(arg); }
+ inline half fabs(half arg) { return unary_specialized<half>::fabs(arg); }
+ inline expr fabs(expr arg) { return unary_specialized<expr>::fabs(arg); }
-/// Absolute value.
-/// \param arg operand
-/// \return absolute value of \a arg
-// template<typename T> typename enable<T,T>::type fabs(T arg) {
-//return unary_specialized<T>::fabs(arg); }
-inline half fabs(half arg) { return unary_specialized<half>::fabs(arg); }
-inline expr fabs(expr arg) { return unary_specialized<expr>::fabs(arg); }
+ /// Remainder of division.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type fmod(T x, U y) { return functions::fmod(x, y); }
+ inline expr fmod(half x, half y) { return functions::fmod(x, y); }
+ inline expr fmod(half x, expr y) { return functions::fmod(x, y); }
+ inline expr fmod(expr x, half y) { return functions::fmod(x, y); }
+ inline expr fmod(expr x, expr y) { return functions::fmod(x, y); }
-/// Remainder of division.
-/// \param x first operand
-/// \param y second operand
-/// \return remainder of floating point division.
-// template<typename T,typename U> typename enable<expr,T,U>::type
-//fmod(T x, U y) { return functions::fmod(x, y); }
-inline expr fmod(half x, half y) { return functions::fmod(x, y); }
-inline expr fmod(half x, expr y) { return functions::fmod(x, y); }
-inline expr fmod(expr x, half y) { return functions::fmod(x, y); }
-inline expr fmod(expr x, expr y) { return functions::fmod(x, y); }
+ /// Remainder of division.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type remainder(T x, U y) { return functions::remainder(x, y); }
+ inline expr remainder(half x, half y) { return functions::remainder(x, y); }
+ inline expr remainder(half x, expr y) { return functions::remainder(x, y); }
+ inline expr remainder(expr x, half y) { return functions::remainder(x, y); }
+ inline expr remainder(expr x, expr y) { return functions::remainder(x, y); }
-/// Remainder of division.
-/// \param x first operand
-/// \param y second operand
-/// \return remainder of floating point division.
-// template<typename T,typename U> typename enable<expr,T,U>::type
-//remainder(T x, U y) { return functions::remainder(x, y); }
-inline expr remainder(half x, half y) { return functions::remainder(x, y); }
-inline expr remainder(half x, expr y) { return functions::remainder(x, y); }
-inline expr remainder(expr x, half y) { return functions::remainder(x, y); }
-inline expr remainder(expr x, expr y) { return functions::remainder(x, y); }
+ /// Remainder of division.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param quo address to store some bits of quotient at
+ /// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type remquo(T x, U y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(half x, half y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(half x, expr y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(expr x, half y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(expr x, expr y, int *quo) { return functions::remquo(x, y, quo); }
-/// Remainder of division.
-/// \param x first operand
-/// \param y second operand
-/// \param quo address to store some bits of quotient at
-/// \return remainder of floating point division.
-// template<typename T,typename U> typename enable<expr,T,U>::type
-//remquo(T x, U y, int *quo) { return functions::remquo(x, y, quo); }
-inline expr remquo(half x, half y, int *quo) {
- return functions::remquo(x, y, quo);
-}
-inline expr remquo(half x, expr y, int *quo) {
- return functions::remquo(x, y, quo);
-}
-inline expr remquo(expr x, half y, int *quo) {
- return functions::remquo(x, y, quo);
-}
-inline expr remquo(expr x, expr y, int *quo) {
- return functions::remquo(x, y, quo);
-}
+ /// Fused multiply add.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param z third operand
+ /// \return ( \a x * \a y ) + \a z rounded as one operation.
+// template<typename T,typename U,typename V> typename enable<expr,T,U,V>::type fma(T x, U y, V z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, half y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, half y, expr z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, expr y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, expr y, expr z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, half y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, half y, expr z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, expr y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, expr y, expr z) { return functions::fma(x, y, z); }
-/// Fused multiply add.
-/// \param x first operand
-/// \param y second operand
-/// \param z third operand
-/// \return ( \a x * \a y ) + \a z rounded as one operation.
-// template<typename T,typename U,typename V> typename
-//enable<expr,T,U,V>::type fma(T x, U y, V z) { return functions::fma(x, y, z);
-//}
-inline expr fma(half x, half y, half z) { return functions::fma(x, y, z); }
-inline expr fma(half x, half y, expr z) { return functions::fma(x, y, z); }
-inline expr fma(half x, expr y, half z) { return functions::fma(x, y, z); }
-inline expr fma(half x, expr y, expr z) { return functions::fma(x, y, z); }
-inline expr fma(expr x, half y, half z) { return functions::fma(x, y, z); }
-inline expr fma(expr x, half y, expr z) { return functions::fma(x, y, z); }
-inline expr fma(expr x, expr y, half z) { return functions::fma(x, y, z); }
-inline expr fma(expr x, expr y, expr z) { return functions::fma(x, y, z); }
+ /// Maximum of half expressions.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return maximum of operands
+// template<typename T,typename U> typename result<T,U>::type fmax(T x, U y) { return binary_specialized<T,U>::fmax(x, y); }
+ inline half fmax(half x, half y) { return binary_specialized<half,half>::fmax(x, y); }
+ inline expr fmax(half x, expr y) { return binary_specialized<half,expr>::fmax(x, y); }
+ inline expr fmax(expr x, half y) { return binary_specialized<expr,half>::fmax(x, y); }
+ inline expr fmax(expr x, expr y) { return binary_specialized<expr,expr>::fmax(x, y); }
-/// Maximum of half expressions.
-/// \param x first operand
-/// \param y second operand
-/// \return maximum of operands
-// template<typename T,typename U> typename result<T,U>::type fmax(T
-//x, U y) { return binary_specialized<T,U>::fmax(x, y); }
-inline half fmax(half x, half y) {
- return binary_specialized<half, half>::fmax(x, y);
-}
-inline expr fmax(half x, expr y) {
- return binary_specialized<half, expr>::fmax(x, y);
-}
-inline expr fmax(expr x, half y) {
- return binary_specialized<expr, half>::fmax(x, y);
-}
-inline expr fmax(expr x, expr y) {
- return binary_specialized<expr, expr>::fmax(x, y);
-}
+ /// Minimum of half expressions.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return minimum of operands
+// template<typename T,typename U> typename result<T,U>::type fmin(T x, U y) { return binary_specialized<T,U>::fmin(x, y); }
+ inline half fmin(half x, half y) { return binary_specialized<half,half>::fmin(x, y); }
+ inline expr fmin(half x, expr y) { return binary_specialized<half,expr>::fmin(x, y); }
+ inline expr fmin(expr x, half y) { return binary_specialized<expr,half>::fmin(x, y); }
+ inline expr fmin(expr x, expr y) { return binary_specialized<expr,expr>::fmin(x, y); }
-/// Minimum of half expressions.
-/// \param x first operand
-/// \param y second operand
-/// \return minimum of operands
-// template<typename T,typename U> typename result<T,U>::type fmin(T
-//x, U y) { return binary_specialized<T,U>::fmin(x, y); }
-inline half fmin(half x, half y) {
- return binary_specialized<half, half>::fmin(x, y);
-}
-inline expr fmin(half x, expr y) {
- return binary_specialized<half, expr>::fmin(x, y);
-}
-inline expr fmin(expr x, half y) {
- return binary_specialized<expr, half>::fmin(x, y);
-}
-inline expr fmin(expr x, expr y) {
- return binary_specialized<expr, expr>::fmin(x, y);
-}
+ /// Positive difference.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return \a x - \a y or 0 if difference negative
+// template<typename T,typename U> typename enable<expr,T,U>::type fdim(T x, U y) { return functions::fdim(x, y); }
+ inline expr fdim(half x, half y) { return functions::fdim(x, y); }
+ inline expr fdim(half x, expr y) { return functions::fdim(x, y); }
+ inline expr fdim(expr x, half y) { return functions::fdim(x, y); }
+ inline expr fdim(expr x, expr y) { return functions::fdim(x, y); }
-/// Positive difference.
-/// \param x first operand
-/// \param y second operand
-/// \return \a x - \a y or 0 if difference negative
-// template<typename T,typename U> typename enable<expr,T,U>::type
-//fdim(T x, U y) { return functions::fdim(x, y); }
-inline expr fdim(half x, half y) { return functions::fdim(x, y); }
-inline expr fdim(half x, expr y) { return functions::fdim(x, y); }
-inline expr fdim(expr x, half y) { return functions::fdim(x, y); }
-inline expr fdim(expr x, expr y) { return functions::fdim(x, y); }
+ /// Get NaN value.
+ /// \return quiet NaN
+ inline half nanh(const char*) { return functions::nanh(); }
-/// Get NaN value.
-/// \return quiet NaN
-inline half nanh(const char *) { return functions::nanh(); }
+ /// \}
+ /// \name Exponential functions
+ /// \{
-/// \}
-/// \name Exponential functions
-/// \{
+ /// Exponential function.
+ /// \param arg function argument
+ /// \return e raised to \a arg
+// template<typename T> typename enable<expr,T>::type exp(T arg) { return functions::exp(arg); }
+ inline expr exp(half arg) { return functions::exp(arg); }
+ inline expr exp(expr arg) { return functions::exp(arg); }
-/// Exponential function.
-/// \param arg function argument
-/// \return e raised to \a arg
-// template<typename T> typename enable<expr,T>::type exp(T arg) {
-//return functions::exp(arg); }
-inline expr exp(half arg) { return functions::exp(arg); }
-inline expr exp(expr arg) { return functions::exp(arg); }
+ /// Exponential minus one.
+ /// \param arg function argument
+ /// \return e raised to \a arg subtracted by 1
+// template<typename T> typename enable<expr,T>::type expm1(T arg) { return functions::expm1(arg); }
+ inline expr expm1(half arg) { return functions::expm1(arg); }
+ inline expr expm1(expr arg) { return functions::expm1(arg); }
-/// Exponential minus one.
-/// \param arg function argument
-/// \return e raised to \a arg subtracted by 1
-// template<typename T> typename enable<expr,T>::type expm1(T arg) {
-//return functions::expm1(arg); }
-inline expr expm1(half arg) { return functions::expm1(arg); }
-inline expr expm1(expr arg) { return functions::expm1(arg); }
+ /// Binary exponential.
+ /// \param arg function argument
+ /// \return 2 raised to \a arg
+// template<typename T> typename enable<expr,T>::type exp2(T arg) { return functions::exp2(arg); }
+ inline expr exp2(half arg) { return functions::exp2(arg); }
+ inline expr exp2(expr arg) { return functions::exp2(arg); }
-/// Binary exponential.
-/// \param arg function argument
-/// \return 2 raised to \a arg
-// template<typename T> typename enable<expr,T>::type exp2(T arg) {
-//return functions::exp2(arg); }
-inline expr exp2(half arg) { return functions::exp2(arg); }
-inline expr exp2(expr arg) { return functions::exp2(arg); }
+ /// Natural logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg to base e
+// template<typename T> typename enable<expr,T>::type log(T arg) { return functions::log(arg); }
+ inline expr log(half arg) { return functions::log(arg); }
+ inline expr log(expr arg) { return functions::log(arg); }
-/// Natural logorithm.
-/// \param arg function argument
-/// \return logarithm of \a arg to base e
-// template<typename T> typename enable<expr,T>::type log(T arg) {
-//return functions::log(arg); }
-inline expr log(half arg) { return functions::log(arg); }
-inline expr log(expr arg) { return functions::log(arg); }
+ /// Common logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg to base 10
+// template<typename T> typename enable<expr,T>::type log10(T arg) { return functions::log10(arg); }
+ inline expr log10(half arg) { return functions::log10(arg); }
+ inline expr log10(expr arg) { return functions::log10(arg); }
-/// Common logorithm.
-/// \param arg function argument
-/// \return logarithm of \a arg to base 10
-// template<typename T> typename enable<expr,T>::type log10(T arg) {
-//return functions::log10(arg); }
-inline expr log10(half arg) { return functions::log10(arg); }
-inline expr log10(expr arg) { return functions::log10(arg); }
+ /// Natural logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg plus 1 to base e
+// template<typename T> typename enable<expr,T>::type log1p(T arg) { return functions::log1p(arg); }
+ inline expr log1p(half arg) { return functions::log1p(arg); }
+ inline expr log1p(expr arg) { return functions::log1p(arg); }
-/// Natural logorithm.
-/// \param arg function argument
-/// \return logarithm of \a arg plus 1 to base e
-// template<typename T> typename enable<expr,T>::type log1p(T arg) {
-//return functions::log1p(arg); }
-inline expr log1p(half arg) { return functions::log1p(arg); }
-inline expr log1p(expr arg) { return functions::log1p(arg); }
+ /// Binary logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg to base 2
+// template<typename T> typename enable<expr,T>::type log2(T arg) { return functions::log2(arg); }
+ inline expr log2(half arg) { return functions::log2(arg); }
+ inline expr log2(expr arg) { return functions::log2(arg); }
-/// Binary logorithm.
-/// \param arg function argument
-/// \return logarithm of \a arg to base 2
-// template<typename T> typename enable<expr,T>::type log2(T arg) {
-//return functions::log2(arg); }
-inline expr log2(half arg) { return functions::log2(arg); }
-inline expr log2(expr arg) { return functions::log2(arg); }
+ /// \}
+ /// \name Power functions
+ /// \{
-/// \}
-/// \name Power functions
-/// \{
+ /// Square root.
+ /// \param arg function argument
+ /// \return square root of \a arg
+// template<typename T> typename enable<expr,T>::type sqrt(T arg) { return functions::sqrt(arg); }
+ inline expr sqrt(half arg) { return functions::sqrt(arg); }
+ inline expr sqrt(expr arg) { return functions::sqrt(arg); }
-/// Square root.
-/// \param arg function argument
-/// \return square root of \a arg
-// template<typename T> typename enable<expr,T>::type sqrt(T arg) {
-//return functions::sqrt(arg); }
-inline expr sqrt(half arg) { return functions::sqrt(arg); }
-inline expr sqrt(expr arg) { return functions::sqrt(arg); }
+ /// Cubic root.
+ /// \param arg function argument
+ /// \return cubic root of \a arg
+// template<typename T> typename enable<expr,T>::type cbrt(T arg) { return functions::cbrt(arg); }
+ inline expr cbrt(half arg) { return functions::cbrt(arg); }
+ inline expr cbrt(expr arg) { return functions::cbrt(arg); }
-/// Cubic root.
-/// \param arg function argument
-/// \return cubic root of \a arg
-// template<typename T> typename enable<expr,T>::type cbrt(T arg) {
-//return functions::cbrt(arg); }
-inline expr cbrt(half arg) { return functions::cbrt(arg); }
-inline expr cbrt(expr arg) { return functions::cbrt(arg); }
+ /// Hypotenuse function.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return square root of sum of squares without internal over- or underflows
+// template<typename T,typename U> typename enable<expr,T,U>::type hypot(T x, U y) { return functions::hypot(x, y); }
+ inline expr hypot(half x, half y) { return functions::hypot(x, y); }
+ inline expr hypot(half x, expr y) { return functions::hypot(x, y); }
+ inline expr hypot(expr x, half y) { return functions::hypot(x, y); }
+ inline expr hypot(expr x, expr y) { return functions::hypot(x, y); }
-/// Hypotenuse function.
-/// \param x first argument
-/// \param y second argument
-/// \return square root of sum of squares without internal over- or underflows
-// template<typename T,typename U> typename enable<expr,T,U>::type
-//hypot(T x, U y) { return functions::hypot(x, y); }
-inline expr hypot(half x, half y) { return functions::hypot(x, y); }
-inline expr hypot(half x, expr y) { return functions::hypot(x, y); }
-inline expr hypot(expr x, half y) { return functions::hypot(x, y); }
-inline expr hypot(expr x, expr y) { return functions::hypot(x, y); }
+ /// Power function.
+ /// \param base first argument
+ /// \param exp second argument
+ /// \return \a base raised to \a exp
+// template<typename T,typename U> typename enable<expr,T,U>::type pow(T base, U exp) { return functions::pow(base, exp); }
+ inline expr pow(half base, half exp) { return functions::pow(base, exp); }
+ inline expr pow(half base, expr exp) { return functions::pow(base, exp); }
+ inline expr pow(expr base, half exp) { return functions::pow(base, exp); }
+ inline expr pow(expr base, expr exp) { return functions::pow(base, exp); }
-/// Power function.
-/// \param base first argument
-/// \param exp second argument
-/// \return \a base raised to \a exp
-// template<typename T,typename U> typename enable<expr,T,U>::type
-//pow(T base, U exp) { return functions::pow(base, exp); }
-inline expr pow(half base, half exp) { return functions::pow(base, exp); }
-inline expr pow(half base, expr exp) { return functions::pow(base, exp); }
-inline expr pow(expr base, half exp) { return functions::pow(base, exp); }
-inline expr pow(expr base, expr exp) { return functions::pow(base, exp); }
+ /// \}
+ /// \name Trigonometric functions
+ /// \{
-/// \}
-/// \name Trigonometric functions
-/// \{
+ /// Sine function.
+ /// \param arg function argument
+ /// \return sine value of \a arg
+// template<typename T> typename enable<expr,T>::type sin(T arg) { return functions::sin(arg); }
+ inline expr sin(half arg) { return functions::sin(arg); }
+ inline expr sin(expr arg) { return functions::sin(arg); }
-/// Sine function.
-/// \param arg function argument
-/// \return sine value of \a arg
-// template<typename T> typename enable<expr,T>::type sin(T arg) {
-//return functions::sin(arg); }
-inline expr sin(half arg) { return functions::sin(arg); }
-inline expr sin(expr arg) { return functions::sin(arg); }
+ /// Cosine function.
+ /// \param arg function argument
+ /// \return cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type cos(T arg) { return functions::cos(arg); }
+ inline expr cos(half arg) { return functions::cos(arg); }
+ inline expr cos(expr arg) { return functions::cos(arg); }
-/// Cosine function.
-/// \param arg function argument
-/// \return cosine value of \a arg
-// template<typename T> typename enable<expr,T>::type cos(T arg) {
-//return functions::cos(arg); }
-inline expr cos(half arg) { return functions::cos(arg); }
-inline expr cos(expr arg) { return functions::cos(arg); }
+ /// Tangent function.
+ /// \param arg function argument
+ /// \return tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type tan(T arg) { return functions::tan(arg); }
+ inline expr tan(half arg) { return functions::tan(arg); }
+ inline expr tan(expr arg) { return functions::tan(arg); }
-/// Tangent function.
-/// \param arg function argument
-/// \return tangent value of \a arg
-// template<typename T> typename enable<expr,T>::type tan(T arg) {
-//return functions::tan(arg); }
-inline expr tan(half arg) { return functions::tan(arg); }
-inline expr tan(expr arg) { return functions::tan(arg); }
+ /// Arc sine.
+ /// \param arg function argument
+ /// \return arc sine value of \a arg
+// template<typename T> typename enable<expr,T>::type asin(T arg) { return functions::asin(arg); }
+ inline expr asin(half arg) { return functions::asin(arg); }
+ inline expr asin(expr arg) { return functions::asin(arg); }
-/// Arc sine.
-/// \param arg function argument
-/// \return arc sine value of \a arg
-// template<typename T> typename enable<expr,T>::type asin(T arg) {
-//return functions::asin(arg); }
-inline expr asin(half arg) { return functions::asin(arg); }
-inline expr asin(expr arg) { return functions::asin(arg); }
+ /// Arc cosine function.
+ /// \param arg function argument
+ /// \return arc cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type acos(T arg) { return functions::acos(arg); }
+ inline expr acos(half arg) { return functions::acos(arg); }
+ inline expr acos(expr arg) { return functions::acos(arg); }
-/// Arc cosine function.
-/// \param arg function argument
-/// \return arc cosine value of \a arg
-// template<typename T> typename enable<expr,T>::type acos(T arg) {
-//return functions::acos(arg); }
-inline expr acos(half arg) { return functions::acos(arg); }
-inline expr acos(expr arg) { return functions::acos(arg); }
+ /// Arc tangent function.
+ /// \param arg function argument
+ /// \return arc tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type atan(T arg) { return functions::atan(arg); }
+ inline expr atan(half arg) { return functions::atan(arg); }
+ inline expr atan(expr arg) { return functions::atan(arg); }
-/// Arc tangent function.
-/// \param arg function argument
-/// \return arc tangent value of \a arg
-// template<typename T> typename enable<expr,T>::type atan(T arg) {
-//return functions::atan(arg); }
-inline expr atan(half arg) { return functions::atan(arg); }
-inline expr atan(expr arg) { return functions::atan(arg); }
+ /// Arc tangent function.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return arc tangent value
+// template<typename T,typename U> typename enable<expr,T,U>::type atan2(T x, U y) { return functions::atan2(x, y); }
+ inline expr atan2(half x, half y) { return functions::atan2(x, y); }
+ inline expr atan2(half x, expr y) { return functions::atan2(x, y); }
+ inline expr atan2(expr x, half y) { return functions::atan2(x, y); }
+ inline expr atan2(expr x, expr y) { return functions::atan2(x, y); }
-/// Arc tangent function.
-/// \param x first argument
-/// \param y second argument
-/// \return arc tangent value
-// template<typename T,typename U> typename enable<expr,T,U>::type
-//atan2(T x, U y) { return functions::atan2(x, y); }
-inline expr atan2(half x, half y) { return functions::atan2(x, y); }
-inline expr atan2(half x, expr y) { return functions::atan2(x, y); }
-inline expr atan2(expr x, half y) { return functions::atan2(x, y); }
-inline expr atan2(expr x, expr y) { return functions::atan2(x, y); }
+ /// \}
+ /// \name Hyperbolic functions
+ /// \{
-/// \}
-/// \name Hyperbolic functions
-/// \{
+ /// Hyperbolic sine.
+ /// \param arg function argument
+ /// \return hyperbolic sine value of \a arg
+// template<typename T> typename enable<expr,T>::type sinh(T arg) { return functions::sinh(arg); }
+ inline expr sinh(half arg) { return functions::sinh(arg); }
+ inline expr sinh(expr arg) { return functions::sinh(arg); }
-/// Hyperbolic sine.
-/// \param arg function argument
-/// \return hyperbolic sine value of \a arg
-// template<typename T> typename enable<expr,T>::type sinh(T arg) {
-//return functions::sinh(arg); }
-inline expr sinh(half arg) { return functions::sinh(arg); }
-inline expr sinh(expr arg) { return functions::sinh(arg); }
+ /// Hyperbolic cosine.
+ /// \param arg function argument
+ /// \return hyperbolic cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type cosh(T arg) { return functions::cosh(arg); }
+ inline expr cosh(half arg) { return functions::cosh(arg); }
+ inline expr cosh(expr arg) { return functions::cosh(arg); }
-/// Hyperbolic cosine.
-/// \param arg function argument
-/// \return hyperbolic cosine value of \a arg
-// template<typename T> typename enable<expr,T>::type cosh(T arg) {
-//return functions::cosh(arg); }
-inline expr cosh(half arg) { return functions::cosh(arg); }
-inline expr cosh(expr arg) { return functions::cosh(arg); }
+ /// Hyperbolic tangent.
+ /// \param arg function argument
+ /// \return hyperbolic tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type tanh(T arg) { return functions::tanh(arg); }
+ inline expr tanh(half arg) { return functions::tanh(arg); }
+ inline expr tanh(expr arg) { return functions::tanh(arg); }
-/// Hyperbolic tangent.
-/// \param arg function argument
-/// \return hyperbolic tangent value of \a arg
-// template<typename T> typename enable<expr,T>::type tanh(T arg) {
-//return functions::tanh(arg); }
-inline expr tanh(half arg) { return functions::tanh(arg); }
-inline expr tanh(expr arg) { return functions::tanh(arg); }
+ /// Hyperbolic area sine.
+ /// \param arg function argument
+ /// \return area sine value of \a arg
+// template<typename T> typename enable<expr,T>::type asinh(T arg) { return functions::asinh(arg); }
+ inline expr asinh(half arg) { return functions::asinh(arg); }
+ inline expr asinh(expr arg) { return functions::asinh(arg); }
-/// Hyperbolic area sine.
-/// \param arg function argument
-/// \return area sine value of \a arg
-// template<typename T> typename enable<expr,T>::type asinh(T arg) {
-//return functions::asinh(arg); }
-inline expr asinh(half arg) { return functions::asinh(arg); }
-inline expr asinh(expr arg) { return functions::asinh(arg); }
+ /// Hyperbolic area cosine.
+ /// \param arg function argument
+ /// \return area cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type acosh(T arg) { return functions::acosh(arg); }
+ inline expr acosh(half arg) { return functions::acosh(arg); }
+ inline expr acosh(expr arg) { return functions::acosh(arg); }
-/// Hyperbolic area cosine.
-/// \param arg function argument
-/// \return area cosine value of \a arg
-// template<typename T> typename enable<expr,T>::type acosh(T arg) {
-//return functions::acosh(arg); }
-inline expr acosh(half arg) { return functions::acosh(arg); }
-inline expr acosh(expr arg) { return functions::acosh(arg); }
+ /// Hyperbolic area tangent.
+ /// \param arg function argument
+ /// \return area tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type atanh(T arg) { return functions::atanh(arg); }
+ inline expr atanh(half arg) { return functions::atanh(arg); }
+ inline expr atanh(expr arg) { return functions::atanh(arg); }
-/// Hyperbolic area tangent.
-/// \param arg function argument
-/// \return area tangent value of \a arg
-// template<typename T> typename enable<expr,T>::type atanh(T arg) {
-//return functions::atanh(arg); }
-inline expr atanh(half arg) { return functions::atanh(arg); }
-inline expr atanh(expr arg) { return functions::atanh(arg); }
+ /// \}
+ /// \name Error and gamma functions
+ /// \{
-/// \}
-/// \name Error and gamma functions
-/// \{
+ /// Error function.
+ /// \param arg function argument
+ /// \return error function value of \a arg
+// template<typename T> typename enable<expr,T>::type erf(T arg) { return functions::erf(arg); }
+ inline expr erf(half arg) { return functions::erf(arg); }
+ inline expr erf(expr arg) { return functions::erf(arg); }
-/// Error function.
-/// \param arg function argument
-/// \return error function value of \a arg
-// template<typename T> typename enable<expr,T>::type erf(T arg) {
-//return functions::erf(arg); }
-inline expr erf(half arg) { return functions::erf(arg); }
-inline expr erf(expr arg) { return functions::erf(arg); }
+ /// Complementary error function.
+ /// \param arg function argument
+ /// \return 1 minus error function value of \a arg
+// template<typename T> typename enable<expr,T>::type erfc(T arg) { return functions::erfc(arg); }
+ inline expr erfc(half arg) { return functions::erfc(arg); }
+ inline expr erfc(expr arg) { return functions::erfc(arg); }
-/// Complementary error function.
-/// \param arg function argument
-/// \return 1 minus error function value of \a arg
-// template<typename T> typename enable<expr,T>::type erfc(T arg) {
-//return functions::erfc(arg); }
-inline expr erfc(half arg) { return functions::erfc(arg); }
-inline expr erfc(expr arg) { return functions::erfc(arg); }
+ /// Natural logarithm of gamma function.
+ /// \param arg function argument
+ /// \return natural logarith of gamma function for \a arg
+// template<typename T> typename enable<expr,T>::type lgamma(T arg) { return functions::lgamma(arg); }
+ inline expr lgamma(half arg) { return functions::lgamma(arg); }
+ inline expr lgamma(expr arg) { return functions::lgamma(arg); }
-/// Natural logarithm of gamma function.
-/// \param arg function argument
-/// \return natural logarith of gamma function for \a arg
-// template<typename T> typename enable<expr,T>::type lgamma(T arg) {
-//return functions::lgamma(arg); }
-inline expr lgamma(half arg) { return functions::lgamma(arg); }
-inline expr lgamma(expr arg) { return functions::lgamma(arg); }
+ /// Gamma function.
+ /// \param arg function argument
+ /// \return gamma function value of \a arg
+// template<typename T> typename enable<expr,T>::type tgamma(T arg) { return functions::tgamma(arg); }
+ inline expr tgamma(half arg) { return functions::tgamma(arg); }
+ inline expr tgamma(expr arg) { return functions::tgamma(arg); }
-/// Gamma function.
-/// \param arg function argument
-/// \return gamma function value of \a arg
-// template<typename T> typename enable<expr,T>::type tgamma(T arg) {
-//return functions::tgamma(arg); }
-inline expr tgamma(half arg) { return functions::tgamma(arg); }
-inline expr tgamma(expr arg) { return functions::tgamma(arg); }
+ /// \}
+ /// \name Rounding
+ /// \{
-/// \}
-/// \name Rounding
-/// \{
+ /// Nearest integer not less than half value.
+ /// \param arg half to round
+ /// \return nearest integer not less than \a arg
+// template<typename T> typename enable<half,T>::type ceil(T arg) { return functions::ceil(arg); }
+ inline half ceil(half arg) { return functions::ceil(arg); }
+ inline half ceil(expr arg) { return functions::ceil(arg); }
-/// Nearest integer not less than half value.
-/// \param arg half to round
-/// \return nearest integer not less than \a arg
-// template<typename T> typename enable<half,T>::type ceil(T arg) {
-//return functions::ceil(arg); }
-inline half ceil(half arg) { return functions::ceil(arg); }
-inline half ceil(expr arg) { return functions::ceil(arg); }
+ /// Nearest integer not greater than half value.
+ /// \param arg half to round
+ /// \return nearest integer not greater than \a arg
+// template<typename T> typename enable<half,T>::type floor(T arg) { return functions::floor(arg); }
+ inline half floor(half arg) { return functions::floor(arg); }
+ inline half floor(expr arg) { return functions::floor(arg); }
-/// Nearest integer not greater than half value.
-/// \param arg half to round
-/// \return nearest integer not greater than \a arg
-// template<typename T> typename enable<half,T>::type floor(T arg) {
-//return functions::floor(arg); }
-inline half floor(half arg) { return functions::floor(arg); }
-inline half floor(expr arg) { return functions::floor(arg); }
+ /// Nearest integer not greater in magnitude than half value.
+ /// \param arg half to round
+ /// \return nearest integer not greater in magnitude than \a arg
+// template<typename T> typename enable<half,T>::type trunc(T arg) { return functions::trunc(arg); }
+ inline half trunc(half arg) { return functions::trunc(arg); }
+ inline half trunc(expr arg) { return functions::trunc(arg); }
-/// Nearest integer not greater in magnitude than half value.
-/// \param arg half to round
-/// \return nearest integer not greater in magnitude than \a arg
-// template<typename T> typename enable<half,T>::type trunc(T arg) {
-//return functions::trunc(arg); }
-inline half trunc(half arg) { return functions::trunc(arg); }
-inline half trunc(expr arg) { return functions::trunc(arg); }
+ /// Nearest integer.
+ /// \param arg half to round
+ /// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<half,T>::type round(T arg) { return functions::round(arg); }
+ inline half round(half arg) { return functions::round(arg); }
+ inline half round(expr arg) { return functions::round(arg); }
-/// Nearest integer.
-/// \param arg half to round
-/// \return nearest integer, rounded away from zero in half-way cases
-// template<typename T> typename enable<half,T>::type round(T arg) {
-//return functions::round(arg); }
-inline half round(half arg) { return functions::round(arg); }
-inline half round(expr arg) { return functions::round(arg); }
+ /// Nearest integer.
+ /// \param arg half to round
+ /// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<long,T>::type lround(T arg) { return functions::lround(arg); }
+ inline long lround(half arg) { return functions::lround(arg); }
+ inline long lround(expr arg) { return functions::lround(arg); }
-/// Nearest integer.
-/// \param arg half to round
-/// \return nearest integer, rounded away from zero in half-way cases
-// template<typename T> typename enable<long,T>::type lround(T arg) {
-//return functions::lround(arg); }
-inline long lround(half arg) { return functions::lround(arg); }
-inline long lround(expr arg) { return functions::lround(arg); }
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<half,T>::type nearbyint(T arg) { return functions::nearbyint(arg); }
+ inline half nearbyint(half arg) { return functions::rint(arg); }
+ inline half nearbyint(expr arg) { return functions::rint(arg); }
-/// Nearest integer using half's internal rounding mode.
-/// \param arg half expression to round
-/// \return nearest integer using default rounding mode
-// template<typename T> typename enable<half,T>::type nearbyint(T
-//arg) { return functions::nearbyint(arg); }
-inline half nearbyint(half arg) { return functions::rint(arg); }
-inline half nearbyint(expr arg) { return functions::rint(arg); }
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<half,T>::type rint(T arg) { return functions::rint(arg); }
+ inline half rint(half arg) { return functions::rint(arg); }
+ inline half rint(expr arg) { return functions::rint(arg); }
-/// Nearest integer using half's internal rounding mode.
-/// \param arg half expression to round
-/// \return nearest integer using default rounding mode
-// template<typename T> typename enable<half,T>::type rint(T arg) {
-//return functions::rint(arg); }
-inline half rint(half arg) { return functions::rint(arg); }
-inline half rint(expr arg) { return functions::rint(arg); }
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<long,T>::type lrint(T arg) { return functions::lrint(arg); }
+ inline long lrint(half arg) { return functions::lrint(arg); }
+ inline long lrint(expr arg) { return functions::lrint(arg); }
+ #if HALF_ENABLE_CPP11_LONG_LONG
+ /// Nearest integer.
+ /// \param arg half to round
+ /// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<long long,T>::type llround(T arg) { return functions::llround(arg); }
+ inline long long llround(half arg) { return functions::llround(arg); }
+ inline long long llround(expr arg) { return functions::llround(arg); }
-/// Nearest integer using half's internal rounding mode.
-/// \param arg half expression to round
-/// \return nearest integer using default rounding mode
-// template<typename T> typename enable<long,T>::type lrint(T arg) {
-//return functions::lrint(arg); }
-inline long lrint(half arg) { return functions::lrint(arg); }
-inline long lrint(expr arg) { return functions::lrint(arg); }
-#if HALF_ENABLE_CPP11_LONG_LONG
-/// Nearest integer.
-/// \param arg half to round
-/// \return nearest integer, rounded away from zero in half-way cases
-// template<typename T> typename enable<long long,T>::type llround(T
-//arg) { return functions::llround(arg); }
-inline long long llround(half arg) { return functions::llround(arg); }
-inline long long llround(expr arg) { return functions::llround(arg); }
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<long long,T>::type llrint(T arg) { return functions::llrint(arg); }
+ inline long long llrint(half arg) { return functions::llrint(arg); }
+ inline long long llrint(expr arg) { return functions::llrint(arg); }
+ #endif
-/// Nearest integer using half's internal rounding mode.
-/// \param arg half expression to round
-/// \return nearest integer using default rounding mode
-// template<typename T> typename enable<long long,T>::type llrint(T
-//arg) { return functions::llrint(arg); }
-inline long long llrint(half arg) { return functions::llrint(arg); }
-inline long long llrint(expr arg) { return functions::llrint(arg); }
-#endif
+ /// \}
+ /// \name Floating point manipulation
+ /// \{
-/// \}
-/// \name Floating point manipulation
-/// \{
+ /// Decompress floating point number.
+ /// \param arg number to decompress
+ /// \param exp address to store exponent at
+ /// \return significant in range [0.5, 1)
+// template<typename T> typename enable<half,T>::type frexp(T arg, int *exp) { return functions::frexp(arg, exp); }
+ inline half frexp(half arg, int *exp) { return functions::frexp(arg, exp); }
+ inline half frexp(expr arg, int *exp) { return functions::frexp(arg, exp); }
-/// Decompress floating point number.
-/// \param arg number to decompress
-/// \param exp address to store exponent at
-/// \return significant in range [0.5, 1)
-// template<typename T> typename enable<half,T>::type frexp(T arg,
-//int *exp) { return functions::frexp(arg, exp); }
-inline half frexp(half arg, int *exp) { return functions::frexp(arg, exp); }
-inline half frexp(expr arg, int *exp) { return functions::frexp(arg, exp); }
+ /// Multiply by power of two.
+ /// \param arg number to modify
+ /// \param exp power of two to multiply with
+ /// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type ldexp(T arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half ldexp(half arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half ldexp(expr arg, int exp) { return functions::scalbln(arg, exp); }
-/// Multiply by power of two.
-/// \param arg number to modify
-/// \param exp power of two to multiply with
-/// \return \a arg multplied by 2 raised to \a exp
-// template<typename T> typename enable<half,T>::type ldexp(T arg,
-//int exp) { return functions::scalbln(arg, exp); }
-inline half ldexp(half arg, int exp) { return functions::scalbln(arg, exp); }
-inline half ldexp(expr arg, int exp) { return functions::scalbln(arg, exp); }
+ /// Extract integer and fractional parts.
+ /// \param arg number to decompress
+ /// \param iptr address to store integer part at
+ /// \return fractional part
+// template<typename T> typename enable<half,T>::type modf(T arg, half *iptr) { return functions::modf(arg, iptr); }
+ inline half modf(half arg, half *iptr) { return functions::modf(arg, iptr); }
+ inline half modf(expr arg, half *iptr) { return functions::modf(arg, iptr); }
-/// Extract integer and fractional parts.
-/// \param arg number to decompress
-/// \param iptr address to store integer part at
-/// \return fractional part
-// template<typename T> typename enable<half,T>::type modf(T arg,
-//half *iptr) { return functions::modf(arg, iptr); }
-inline half modf(half arg, half *iptr) { return functions::modf(arg, iptr); }
-inline half modf(expr arg, half *iptr) { return functions::modf(arg, iptr); }
+ /// Multiply by power of two.
+ /// \param arg number to modify
+ /// \param exp power of two to multiply with
+ /// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type scalbn(T arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half scalbn(half arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half scalbn(expr arg, int exp) { return functions::scalbln(arg, exp); }
-/// Multiply by power of two.
-/// \param arg number to modify
-/// \param exp power of two to multiply with
-/// \return \a arg multplied by 2 raised to \a exp
-// template<typename T> typename enable<half,T>::type scalbn(T arg,
-//int exp) { return functions::scalbln(arg, exp); }
-inline half scalbn(half arg, int exp) { return functions::scalbln(arg, exp); }
-inline half scalbn(expr arg, int exp) { return functions::scalbln(arg, exp); }
+ /// Multiply by power of two.
+ /// \param arg number to modify
+ /// \param exp power of two to multiply with
+ /// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type scalbln(T arg, long exp) { return functions::scalbln(arg, exp); }
+ inline half scalbln(half arg, long exp) { return functions::scalbln(arg, exp); }
+ inline half scalbln(expr arg, long exp) { return functions::scalbln(arg, exp); }
-/// Multiply by power of two.
-/// \param arg number to modify
-/// \param exp power of two to multiply with
-/// \return \a arg multplied by 2 raised to \a exp
-// template<typename T> typename enable<half,T>::type scalbln(T arg,
-//long exp) { return functions::scalbln(arg, exp); }
-inline half scalbln(half arg, long exp) { return functions::scalbln(arg, exp); }
-inline half scalbln(expr arg, long exp) { return functions::scalbln(arg, exp); }
+ /// Extract exponent.
+ /// \param arg number to query
+ /// \return floating point exponent
+ /// \retval FP_ILOGB0 for zero
+ /// \retval FP_ILOGBNAN for NaN
+ /// \retval MAX_INT for infinity
+// template<typename T> typename enable<int,T>::type ilogb(T arg) { return functions::ilogb(arg); }
+ inline int ilogb(half arg) { return functions::ilogb(arg); }
+ inline int ilogb(expr arg) { return functions::ilogb(arg); }
-/// Extract exponent.
-/// \param arg number to query
-/// \return floating point exponent
-/// \retval FP_ILOGB0 for zero
-/// \retval FP_ILOGBNAN for NaN
-/// \retval MAX_INT for infinity
-// template<typename T> typename enable<int,T>::type ilogb(T arg) {
-//return functions::ilogb(arg); }
-inline int ilogb(half arg) { return functions::ilogb(arg); }
-inline int ilogb(expr arg) { return functions::ilogb(arg); }
+ /// Extract exponent.
+ /// \param arg number to query
+ /// \return floating point exponent
+// template<typename T> typename enable<half,T>::type logb(T arg) { return functions::logb(arg); }
+ inline half logb(half arg) { return functions::logb(arg); }
+ inline half logb(expr arg) { return functions::logb(arg); }
-/// Extract exponent.
-/// \param arg number to query
-/// \return floating point exponent
-// template<typename T> typename enable<half,T>::type logb(T arg) {
-//return functions::logb(arg); }
-inline half logb(half arg) { return functions::logb(arg); }
-inline half logb(expr arg) { return functions::logb(arg); }
+ /// Next representable value.
+ /// \param from value to compute next representable value for
+ /// \param to direction towards which to compute next value
+ /// \return next representable value after \a from in direction towards \a to
+// template<typename T,typename U> typename enable<half,T,U>::type nextafter(T from, U to) { return functions::nextafter(from, to); }
+ inline half nextafter(half from, half to) { return functions::nextafter(from, to); }
+ inline half nextafter(half from, expr to) { return functions::nextafter(from, to); }
+ inline half nextafter(expr from, half to) { return functions::nextafter(from, to); }
+ inline half nextafter(expr from, expr to) { return functions::nextafter(from, to); }
-/// Next representable value.
-/// \param from value to compute next representable value for
-/// \param to direction towards which to compute next value
-/// \return next representable value after \a from in direction towards \a to
-// template<typename T,typename U> typename enable<half,T,U>::type
-//nextafter(T from, U to) { return functions::nextafter(from, to); }
-inline half nextafter(half from, half to) {
- return functions::nextafter(from, to);
-}
-inline half nextafter(half from, expr to) {
- return functions::nextafter(from, to);
-}
-inline half nextafter(expr from, half to) {
- return functions::nextafter(from, to);
-}
-inline half nextafter(expr from, expr to) {
- return functions::nextafter(from, to);
-}
+ /// Next representable value.
+ /// \param from value to compute next representable value for
+ /// \param to direction towards which to compute next value
+ /// \return next representable value after \a from in direction towards \a to
+// template<typename T> typename enable<half,T>::type nexttoward(T from, long double to) { return functions::nexttoward(from, to); }
+ inline half nexttoward(half from, long double to) { return functions::nexttoward(from, to); }
+ inline half nexttoward(expr from, long double to) { return functions::nexttoward(from, to); }
-/// Next representable value.
-/// \param from value to compute next representable value for
-/// \param to direction towards which to compute next value
-/// \return next representable value after \a from in direction towards \a to
-// template<typename T> typename enable<half,T>::type nexttoward(T
-//from, long double to) { return functions::nexttoward(from, to); }
-inline half nexttoward(half from, long double to) {
- return functions::nexttoward(from, to);
-}
-inline half nexttoward(expr from, long double to) {
- return functions::nexttoward(from, to);
-}
+ /// Take sign.
+ /// \param x value to change sign for
+ /// \param y value to take sign from
+ /// \return value equal to \a x in magnitude and to \a y in sign
+// template<typename T,typename U> typename enable<half,T,U>::type copysign(T x, U y) { return functions::copysign(x, y); }
+ inline half copysign(half x, half y) { return functions::copysign(x, y); }
+ inline half copysign(half x, expr y) { return functions::copysign(x, y); }
+ inline half copysign(expr x, half y) { return functions::copysign(x, y); }
+ inline half copysign(expr x, expr y) { return functions::copysign(x, y); }
-/// Take sign.
-/// \param x value to change sign for
-/// \param y value to take sign from
-/// \return value equal to \a x in magnitude and to \a y in sign
-// template<typename T,typename U> typename enable<half,T,U>::type
-//copysign(T x, U y) { return functions::copysign(x, y); }
-inline half copysign(half x, half y) { return functions::copysign(x, y); }
-inline half copysign(half x, expr y) { return functions::copysign(x, y); }
-inline half copysign(expr x, half y) { return functions::copysign(x, y); }
-inline half copysign(expr x, expr y) { return functions::copysign(x, y); }
+ /// \}
+ /// \name Floating point classification
+ /// \{
-/// \}
-/// \name Floating point classification
-/// \{
-/// Classify floating point value.
-/// \param arg number to classify
-/// \retval FP_ZERO for positive and negative zero
-/// \retval FP_SUBNORMAL for subnormal numbers
-/// \retval FP_INFINITY for positive and negative infinity
-/// \retval FP_NAN for NaNs
-/// \retval FP_NORMAL for all other (normal) values
-// template<typename T> typename enable<int,T>::type fpclassify(T
-//arg) { return functions::fpclassify(arg); }
-inline int fpclassify(half arg) { return functions::fpclassify(arg); }
-inline int fpclassify(expr arg) { return functions::fpclassify(arg); }
+ /// Classify floating point value.
+ /// \param arg number to classify
+ /// \retval FP_ZERO for positive and negative zero
+ /// \retval FP_SUBNORMAL for subnormal numbers
+ /// \retval FP_INFINITY for positive and negative infinity
+ /// \retval FP_NAN for NaNs
+ /// \retval FP_NORMAL for all other (normal) values
+// template<typename T> typename enable<int,T>::type fpclassify(T arg) { return functions::fpclassify(arg); }
+ inline int fpclassify(half arg) { return functions::fpclassify(arg); }
+ inline int fpclassify(expr arg) { return functions::fpclassify(arg); }
-/// Check if finite number.
-/// \param arg number to check
-/// \retval true if neither infinity nor NaN
-/// \retval false else
-// template<typename T> typename enable<bool,T>::type isfinite(T arg)
-//{ return functions::isfinite(arg); }
-inline bool isfinite(half arg) { return functions::isfinite(arg); }
-inline bool isfinite(expr arg) { return functions::isfinite(arg); }
+ /// Check if finite number.
+ /// \param arg number to check
+ /// \retval true if neither infinity nor NaN
+ /// \retval false else
+// template<typename T> typename enable<bool,T>::type isfinite(T arg) { return functions::isfinite(arg); }
+ inline bool isfinite(half arg) { return functions::isfinite(arg); }
+ inline bool isfinite(expr arg) { return functions::isfinite(arg); }
-/// Check for infinity.
-/// \param arg number to check
-/// \retval true for positive or negative infinity
-/// \retval false else
-// template<typename T> typename enable<bool,T>::type isinf(T arg) {
-//return functions::isinf(arg); }
-inline bool isinf(half arg) { return functions::isinf(arg); }
-inline bool isinf(expr arg) { return functions::isinf(arg); }
+ /// Check for infinity.
+ /// \param arg number to check
+ /// \retval true for positive or negative infinity
+ /// \retval false else
+// template<typename T> typename enable<bool,T>::type isinf(T arg) { return functions::isinf(arg); }
+ inline bool isinf(half arg) { return functions::isinf(arg); }
+ inline bool isinf(expr arg) { return functions::isinf(arg); }
-/// Check for NaN.
-/// \param arg number to check
-/// \retval true for NaNs
-/// \retval false else
-// template<typename T> typename enable<bool,T>::type isnan(T arg) {
-//return functions::isnan(arg); }
-inline bool isnan(half arg) { return functions::isnan(arg); }
-inline bool isnan(expr arg) { return functions::isnan(arg); }
+ /// Check for NaN.
+ /// \param arg number to check
+ /// \retval true for NaNs
+ /// \retval false else
+// template<typename T> typename enable<bool,T>::type isnan(T arg) { return functions::isnan(arg); }
+ inline bool isnan(half arg) { return functions::isnan(arg); }
+ inline bool isnan(expr arg) { return functions::isnan(arg); }
-/// Check if normal number.
-/// \param arg number to check
-/// \retval true if normal number
-/// \retval false if either subnormal, zero, infinity or NaN
-// template<typename T> typename enable<bool,T>::type isnormal(T arg)
-//{ return functions::isnormal(arg); }
-inline bool isnormal(half arg) { return functions::isnormal(arg); }
-inline bool isnormal(expr arg) { return functions::isnormal(arg); }
+ /// Check if normal number.
+ /// \param arg number to check
+ /// \retval true if normal number
+ /// \retval false if either subnormal, zero, infinity or NaN
+// template<typename T> typename enable<bool,T>::type isnormal(T arg) { return functions::isnormal(arg); }
+ inline bool isnormal(half arg) { return functions::isnormal(arg); }
+ inline bool isnormal(expr arg) { return functions::isnormal(arg); }
-/// Check sign.
-/// \param arg number to check
-/// \retval true for negative number
-/// \retval false for positive number
-// template<typename T> typename enable<bool,T>::type signbit(T arg)
-//{ return functions::signbit(arg); }
-inline bool signbit(half arg) { return functions::signbit(arg); }
-inline bool signbit(expr arg) { return functions::signbit(arg); }
+ /// Check sign.
+ /// \param arg number to check
+ /// \retval true for negative number
+ /// \retval false for positive number
+// template<typename T> typename enable<bool,T>::type signbit(T arg) { return functions::signbit(arg); }
+ inline bool signbit(half arg) { return functions::signbit(arg); }
+ inline bool signbit(expr arg) { return functions::signbit(arg); }
-/// \}
-/// \name Comparison
-/// \{
+ /// \}
+ /// \name Comparison
+ /// \{
-/// Comparison for greater than.
-/// \param x first operand
-/// \param y second operand
-/// \retval true if \a x greater than \a y
-/// \retval false else
-// template<typename T,typename U> typename enable<bool,T,U>::type
-//isgreater(T x, U y) { return functions::isgreater(x, y); }
-inline bool isgreater(half x, half y) { return functions::isgreater(x, y); }
-inline bool isgreater(half x, expr y) { return functions::isgreater(x, y); }
-inline bool isgreater(expr x, half y) { return functions::isgreater(x, y); }
-inline bool isgreater(expr x, expr y) { return functions::isgreater(x, y); }
+ /// Comparison for greater than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater than \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isgreater(T x, U y) { return functions::isgreater(x, y); }
+ inline bool isgreater(half x, half y) { return functions::isgreater(x, y); }
+ inline bool isgreater(half x, expr y) { return functions::isgreater(x, y); }
+ inline bool isgreater(expr x, half y) { return functions::isgreater(x, y); }
+ inline bool isgreater(expr x, expr y) { return functions::isgreater(x, y); }
-/// Comparison for greater equal.
-/// \param x first operand
-/// \param y second operand
-/// \retval true if \a x greater equal \a y
-/// \retval false else
-// template<typename T,typename U> typename enable<bool,T,U>::type
-//isgreaterequal(T x, U y) { return functions::isgreaterequal(x, y); }
-inline bool isgreaterequal(half x, half y) {
- return functions::isgreaterequal(x, y);
-}
-inline bool isgreaterequal(half x, expr y) {
- return functions::isgreaterequal(x, y);
-}
-inline bool isgreaterequal(expr x, half y) {
- return functions::isgreaterequal(x, y);
-}
-inline bool isgreaterequal(expr x, expr y) {
- return functions::isgreaterequal(x, y);
-}
+ /// Comparison for greater equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater equal \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isgreaterequal(T x, U y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(half x, half y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(half x, expr y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(expr x, half y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(expr x, expr y) { return functions::isgreaterequal(x, y); }
-/// Comparison for less than.
-/// \param x first operand
-/// \param y second operand
-/// \retval true if \a x less than \a y
-/// \retval false else
-// template<typename T,typename U> typename enable<bool,T,U>::type
-//isless(T x, U y) { return functions::isless(x, y); }
-inline bool isless(half x, half y) { return functions::isless(x, y); }
-inline bool isless(half x, expr y) { return functions::isless(x, y); }
-inline bool isless(expr x, half y) { return functions::isless(x, y); }
-inline bool isless(expr x, expr y) { return functions::isless(x, y); }
+ /// Comparison for less than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less than \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isless(T x, U y) { return functions::isless(x, y); }
+ inline bool isless(half x, half y) { return functions::isless(x, y); }
+ inline bool isless(half x, expr y) { return functions::isless(x, y); }
+ inline bool isless(expr x, half y) { return functions::isless(x, y); }
+ inline bool isless(expr x, expr y) { return functions::isless(x, y); }
-/// Comparison for less equal.
-/// \param x first operand
-/// \param y second operand
-/// \retval true if \a x less equal \a y
-/// \retval false else
-// template<typename T,typename U> typename enable<bool,T,U>::type
-//islessequal(T x, U y) { return functions::islessequal(x, y); }
-inline bool islessequal(half x, half y) { return functions::islessequal(x, y); }
-inline bool islessequal(half x, expr y) { return functions::islessequal(x, y); }
-inline bool islessequal(expr x, half y) { return functions::islessequal(x, y); }
-inline bool islessequal(expr x, expr y) { return functions::islessequal(x, y); }
+ /// Comparison for less equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less equal \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type islessequal(T x, U y) { return functions::islessequal(x, y); }
+ inline bool islessequal(half x, half y) { return functions::islessequal(x, y); }
+ inline bool islessequal(half x, expr y) { return functions::islessequal(x, y); }
+ inline bool islessequal(expr x, half y) { return functions::islessequal(x, y); }
+ inline bool islessequal(expr x, expr y) { return functions::islessequal(x, y); }
-/// Comarison for less or greater.
-/// \param x first operand
-/// \param y second operand
-/// \retval true if either less or greater
-/// \retval false else
-// template<typename T,typename U> typename enable<bool,T,U>::type
-//islessgreater(T x, U y) { return functions::islessgreater(x, y); }
-inline bool islessgreater(half x, half y) {
- return functions::islessgreater(x, y);
-}
-inline bool islessgreater(half x, expr y) {
- return functions::islessgreater(x, y);
-}
-inline bool islessgreater(expr x, half y) {
- return functions::islessgreater(x, y);
-}
-inline bool islessgreater(expr x, expr y) {
- return functions::islessgreater(x, y);
-}
+ /// Comarison for less or greater.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if either less or greater
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type islessgreater(T x, U y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(half x, half y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(half x, expr y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(expr x, half y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(expr x, expr y) { return functions::islessgreater(x, y); }
-/// Check if unordered.
-/// \param x first operand
-/// \param y second operand
-/// \retval true if unordered (one or two NaN operands)
-/// \retval false else
-// template<typename T,typename U> typename enable<bool,T,U>::type
-//isunordered(T x, U y) { return functions::isunordered(x, y); }
-inline bool isunordered(half x, half y) { return functions::isunordered(x, y); }
-inline bool isunordered(half x, expr y) { return functions::isunordered(x, y); }
-inline bool isunordered(expr x, half y) { return functions::isunordered(x, y); }
-inline bool isunordered(expr x, expr y) { return functions::isunordered(x, y); }
+ /// Check if unordered.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if unordered (one or two NaN operands)
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isunordered(T x, U y) { return functions::isunordered(x, y); }
+ inline bool isunordered(half x, half y) { return functions::isunordered(x, y); }
+ inline bool isunordered(half x, expr y) { return functions::isunordered(x, y); }
+ inline bool isunordered(expr x, half y) { return functions::isunordered(x, y); }
+ inline bool isunordered(expr x, expr y) { return functions::isunordered(x, y); }
-/// \name Casting
-/// \{
+ /// \name Casting
+ /// \{
-/// Cast to or from half-precision floating point number.
-/// This casts between [half](\ref half_float::half) and any built-in arithmetic
-/// type. The values are converted
-/// directly using the given rounding mode, without any roundtrip over `float`
-/// that a `static_cast` would otherwise do.
-/// It uses the default rounding mode.
-///
-/// Using this cast with neither of the two types being a [half](\ref
-/// half_float::half) or with any of the two types
-/// not being a built-in arithmetic type (apart from [half](\ref
-/// half_float::half), of course) results in a compiler
-/// error and casting between [half](\ref half_float::half)s is just a no-op.
-/// \tparam T destination type (half or built-in arithmetic type)
-/// \tparam U source type (half or built-in arithmetic type)
-/// \param arg value to cast
-/// \return \a arg converted to destination type
-template <typename T, typename U>
-T half_cast(U arg) {
- return half_caster<T, U>::cast(arg);
-}
+ /// Cast to or from half-precision floating point number.
+ /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted
+ /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do.
+ /// It uses the default rounding mode.
+ ///
+ /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types
+ /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler
+ /// error and casting between [half](\ref half_float::half)s is just a no-op.
+ /// \tparam T destination type (half or built-in arithmetic type)
+ /// \tparam U source type (half or built-in arithmetic type)
+ /// \param arg value to cast
+ /// \return \a arg converted to destination type
+ template<typename T,typename U> T half_cast(U arg) { return half_caster<T,U>::cast(arg); }
-/// Cast to or from half-precision floating point number.
-/// This casts between [half](\ref half_float::half) and any built-in arithmetic
-/// type. The values are converted
-/// directly using the given rounding mode, without any roundtrip over `float`
-/// that a `static_cast` would otherwise do.
-///
-/// Using this cast with neither of the two types being a [half](\ref
-/// half_float::half) or with any of the two types
-/// not being a built-in arithmetic type (apart from [half](\ref
-/// half_float::half), of course) results in a compiler
-/// error and casting between [half](\ref half_float::half)s is just a no-op.
-/// \tparam T destination type (half or built-in arithmetic type)
-/// \tparam R rounding mode to use.
-/// \tparam U source type (half or built-in arithmetic type)
-/// \param arg value to cast
-/// \return \a arg converted to destination type
-template <typename T, std::float_round_style R, typename U>
-T half_cast(U arg) {
- return half_caster<T, U, R>::cast(arg);
-}
-/// \}
-}
+ /// Cast to or from half-precision floating point number.
+ /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted
+ /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do.
+ ///
+ /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types
+ /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler
+ /// error and casting between [half](\ref half_float::half)s is just a no-op.
+ /// \tparam T destination type (half or built-in arithmetic type)
+ /// \tparam R rounding mode to use.
+ /// \tparam U source type (half or built-in arithmetic type)
+ /// \param arg value to cast
+ /// \return \a arg converted to destination type
+ template<typename T,std::float_round_style R,typename U> T half_cast(U arg) { return half_caster<T,U,R>::cast(arg); }
+ /// \}
+ }
-using detail::operator==;
-using detail::operator!=;
-using detail::operator<;
-using detail::operator>;
-using detail::operator<=;
-using detail::operator>=;
-using detail::operator+;
-using detail::operator-;
-using detail::operator*;
-using detail::operator/;
-using detail::operator<<;
-using detail::operator>>;
+ using detail::operator==;
+ using detail::operator!=;
+ using detail::operator<;
+ using detail::operator>;
+ using detail::operator<=;
+ using detail::operator>=;
+ using detail::operator+;
+ using detail::operator-;
+ using detail::operator*;
+ using detail::operator/;
+ using detail::operator<<;
+ using detail::operator>>;
-using detail::abs;
-using detail::fabs;
-using detail::fmod;
-using detail::remainder;
-using detail::remquo;
-using detail::fma;
-using detail::fmax;
-using detail::fmin;
-using detail::fdim;
-using detail::nanh;
-using detail::exp;
-using detail::expm1;
-using detail::exp2;
-using detail::log;
-using detail::log10;
-using detail::log1p;
-using detail::log2;
-using detail::sqrt;
-using detail::cbrt;
-using detail::hypot;
-using detail::pow;
-using detail::sin;
-using detail::cos;
-using detail::tan;
-using detail::asin;
-using detail::acos;
-using detail::atan;
-using detail::atan2;
-using detail::sinh;
-using detail::cosh;
-using detail::tanh;
-using detail::asinh;
-using detail::acosh;
-using detail::atanh;
-using detail::erf;
-using detail::erfc;
-using detail::lgamma;
-using detail::tgamma;
-using detail::ceil;
-using detail::floor;
-using detail::trunc;
-using detail::round;
-using detail::lround;
-using detail::nearbyint;
-using detail::rint;
-using detail::lrint;
+ using detail::abs;
+ using detail::fabs;
+ using detail::fmod;
+ using detail::remainder;
+ using detail::remquo;
+ using detail::fma;
+ using detail::fmax;
+ using detail::fmin;
+ using detail::fdim;
+ using detail::nanh;
+ using detail::exp;
+ using detail::expm1;
+ using detail::exp2;
+ using detail::log;
+ using detail::log10;
+ using detail::log1p;
+ using detail::log2;
+ using detail::sqrt;
+ using detail::cbrt;
+ using detail::hypot;
+ using detail::pow;
+ using detail::sin;
+ using detail::cos;
+ using detail::tan;
+ using detail::asin;
+ using detail::acos;
+ using detail::atan;
+ using detail::atan2;
+ using detail::sinh;
+ using detail::cosh;
+ using detail::tanh;
+ using detail::asinh;
+ using detail::acosh;
+ using detail::atanh;
+ using detail::erf;
+ using detail::erfc;
+ using detail::lgamma;
+ using detail::tgamma;
+ using detail::ceil;
+ using detail::floor;
+ using detail::trunc;
+ using detail::round;
+ using detail::lround;
+ using detail::nearbyint;
+ using detail::rint;
+ using detail::lrint;
#if HALF_ENABLE_CPP11_LONG_LONG
-using detail::llround;
-using detail::llrint;
+ using detail::llround;
+ using detail::llrint;
#endif
-using detail::frexp;
-using detail::ldexp;
-using detail::modf;
-using detail::scalbn;
-using detail::scalbln;
-using detail::ilogb;
-using detail::logb;
-using detail::nextafter;
-using detail::nexttoward;
-using detail::copysign;
-using detail::fpclassify;
-using detail::isfinite;
-using detail::isinf;
-using detail::isnan;
-using detail::isnormal;
-using detail::signbit;
-using detail::isgreater;
-using detail::isgreaterequal;
-using detail::isless;
-using detail::islessequal;
-using detail::islessgreater;
-using detail::isunordered;
+ using detail::frexp;
+ using detail::ldexp;
+ using detail::modf;
+ using detail::scalbn;
+ using detail::scalbln;
+ using detail::ilogb;
+ using detail::logb;
+ using detail::nextafter;
+ using detail::nexttoward;
+ using detail::copysign;
+ using detail::fpclassify;
+ using detail::isfinite;
+ using detail::isinf;
+ using detail::isnan;
+ using detail::isnormal;
+ using detail::signbit;
+ using detail::isgreater;
+ using detail::isgreaterequal;
+ using detail::isless;
+ using detail::islessequal;
+ using detail::islessgreater;
+ using detail::isunordered;
-using detail::half_cast;
+ using detail::half_cast;
}
+
/// Extensions to the C++ standard library.
-namespace std {
-/// Numeric limits for half-precision floats.
-/// Because of the underlying single-precision implementation of many
-/// operations, it inherits some properties from
-/// `std::numeric_limits<float>`.
-template <>
-class numeric_limits<half_float::half> : public numeric_limits<float> {
- public:
- /// Supports signed values.
- static HALF_CONSTEXPR_CONST bool is_signed = true;
+namespace std
+{
+ /// Numeric limits for half-precision floats.
+ /// Because of the underlying single-precision implementation of many operations, it inherits some properties from
+ /// `std::numeric_limits<float>`.
+ template<> class numeric_limits<half_float::half> : public numeric_limits<float>
+ {
+ public:
+ /// Supports signed values.
+ static HALF_CONSTEXPR_CONST bool is_signed = true;
- /// Is not exact.
- static HALF_CONSTEXPR_CONST bool is_exact = false;
+ /// Is not exact.
+ static HALF_CONSTEXPR_CONST bool is_exact = false;
- /// Doesn't provide modulo arithmetic.
- static HALF_CONSTEXPR_CONST bool is_modulo = false;
+ /// Doesn't provide modulo arithmetic.
+ static HALF_CONSTEXPR_CONST bool is_modulo = false;
- /// IEEE conformant.
- static HALF_CONSTEXPR_CONST bool is_iec559 = true;
+ /// IEEE conformant.
+ static HALF_CONSTEXPR_CONST bool is_iec559 = true;
- /// Supports infinity.
- static HALF_CONSTEXPR_CONST bool has_infinity = true;
+ /// Supports infinity.
+ static HALF_CONSTEXPR_CONST bool has_infinity = true;
- /// Supports quiet NaNs.
- static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true;
+ /// Supports quiet NaNs.
+ static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true;
- /// Supports subnormal values.
- static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present;
+ /// Supports subnormal values.
+ static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present;
- /// Rounding mode.
- /// Due to the mix of internal single-precision computations (using the
- /// rounding mode of the underlying
- /// single-precision implementation) with the rounding mode of the
- /// single-to-half conversions, the actual rounding
- /// mode might be `std::round_indeterminate` if the default half-precision
- /// rounding mode doesn't match the
- /// single-precision rounding mode.
- static HALF_CONSTEXPR_CONST float_round_style round_style =
- (std::numeric_limits<float>::round_style == half_float::half::round_style)
- ? half_float::half::round_style
- : round_indeterminate;
+ /// Rounding mode.
+ /// Due to the mix of internal single-precision computations (using the rounding mode of the underlying
+ /// single-precision implementation) with the rounding mode of the single-to-half conversions, the actual rounding
+ /// mode might be `std::round_indeterminate` if the default half-precision rounding mode doesn't match the
+ /// single-precision rounding mode.
+ static HALF_CONSTEXPR_CONST float_round_style round_style = (std::numeric_limits<float>::round_style==
+ half_float::half::round_style) ? half_float::half::round_style : round_indeterminate;
- /// Significant digits.
- static HALF_CONSTEXPR_CONST int digits = 11;
+ /// Significant digits.
+ static HALF_CONSTEXPR_CONST int digits = 11;
- /// Significant decimal digits.
- static HALF_CONSTEXPR_CONST int digits10 = 3;
+ /// Significant decimal digits.
+ static HALF_CONSTEXPR_CONST int digits10 = 3;
- /// Required decimal digits to represent all possible values.
- static HALF_CONSTEXPR_CONST int max_digits10 = 5;
+ /// Required decimal digits to represent all possible values.
+ static HALF_CONSTEXPR_CONST int max_digits10 = 5;
- /// Number base.
- static HALF_CONSTEXPR_CONST int radix = 2;
+ /// Number base.
+ static HALF_CONSTEXPR_CONST int radix = 2;
- /// One more than smallest exponent.
- static HALF_CONSTEXPR_CONST int min_exponent = -13;
+ /// One more than smallest exponent.
+ static HALF_CONSTEXPR_CONST int min_exponent = -13;
- /// Smallest normalized representable power of 10.
- static HALF_CONSTEXPR_CONST int min_exponent10 = -4;
+ /// Smallest normalized representable power of 10.
+ static HALF_CONSTEXPR_CONST int min_exponent10 = -4;
- /// One more than largest exponent
- static HALF_CONSTEXPR_CONST int max_exponent = 16;
+ /// One more than largest exponent
+ static HALF_CONSTEXPR_CONST int max_exponent = 16;
- /// Largest finitely representable power of 10.
- static HALF_CONSTEXPR_CONST int max_exponent10 = 4;
+ /// Largest finitely representable power of 10.
+ static HALF_CONSTEXPR_CONST int max_exponent10 = 4;
- /// Smallest positive normal value.
- static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW {
- return half_float::half(half_float::detail::binary, 0x0400);
- }
+ /// Smallest positive normal value.
+ static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0400); }
- /// Smallest finite value.
- static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW {
- return half_float::half(half_float::detail::binary, 0xFBFF);
- }
+ /// Smallest finite value.
+ static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0xFBFF); }
- /// Largest finite value.
- static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW {
- return half_float::half(half_float::detail::binary, 0x7BFF);
- }
+ /// Largest finite value.
+ static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7BFF); }
- /// Difference between one and next representable value.
- static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW {
- return half_float::half(half_float::detail::binary, 0x1400);
- }
+ /// Difference between one and next representable value.
+ static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x1400); }
- /// Maximum rounding error.
- static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW {
- return half_float::half(
- half_float::detail::binary,
- (round_style == std::round_to_nearest) ? 0x3800 : 0x3C00);
- }
+ /// Maximum rounding error.
+ static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW
+ { return half_float::half(half_float::detail::binary, (round_style==std::round_to_nearest) ? 0x3800 : 0x3C00); }
- /// Positive infinity.
- static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW {
- return half_float::half(half_float::detail::binary, 0x7C00);
- }
+ /// Positive infinity.
+ static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7C00); }
- /// Quiet NaN.
- static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW {
- return half_float::half(half_float::detail::binary, 0x7FFF);
- }
+ /// Quiet NaN.
+ static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7FFF); }
- /// Signalling NaN.
- static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW {
- return half_float::half(half_float::detail::binary, 0x7DFF);
- }
+ /// Signalling NaN.
+ static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7DFF); }
- /// Smallest positive subnormal value.
- static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW {
- return half_float::half(half_float::detail::binary, 0x0001);
- }
-};
+ /// Smallest positive subnormal value.
+ static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0001); }
+ };
#if HALF_ENABLE_CPP11_HASH
-/// Hash function for half-precision floats.
-/// This is only defined if C++11 `std::hash` is supported and enabled.
-template <>
-struct hash<half_float::half> //: unary_function<half_float::half,size_t>
-{
- /// Type of function argument.
- typedef half_float::half argument_type;
+ /// Hash function for half-precision floats.
+ /// This is only defined if C++11 `std::hash` is supported and enabled.
+ template<> struct hash<half_float::half> //: unary_function<half_float::half,size_t>
+ {
+ /// Type of function argument.
+ typedef half_float::half argument_type;
- /// Function return type.
- typedef size_t result_type;
+ /// Function return type.
+ typedef size_t result_type;
- /// Compute hash function.
- /// \param arg half to hash
- /// \return hash value
- result_type operator()(argument_type arg) const {
- return hash<half_float::detail::uint16>()(static_cast<unsigned>(arg.data_) &
- -(arg.data_ != 0x8000));
- }
-};
+ /// Compute hash function.
+ /// \param arg half to hash
+ /// \return hash value
+ result_type operator()(argument_type arg) const
+ { return hash<half_float::detail::uint16>()(static_cast<unsigned>(arg.data_)&-(arg.data_!=0x8000)); }
+ };
#endif
}
+
#undef HALF_CONSTEXPR
#undef HALF_CONSTEXPR_CONST
#undef HALF_NOEXCEPT
#undef HALF_NOTHROW
#ifdef HALF_POP_WARNINGS
-#pragma warning(pop)
-#undef HALF_POP_WARNINGS
+ #pragma warning(pop)
+ #undef HALF_POP_WARNINGS
#endif
#endif
diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 861f3bf..014e588 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -8,16 +8,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -27,34 +25,34 @@ // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#include "instructions.h" #include "half.h" #include "half.hpp" -#include "opcodes.h" +#include "instructions.h" #include "ptx_ir.h" +#include "opcodes.h" #include "ptx_sim.h" -typedef void *yyscan_t; +typedef void * yyscan_t; class ptx_recognizer; -#include <assert.h> -#include <fenv.h> -#include <math.h> -#include <stdio.h> -#include <stdlib.h> +#include "ptx.tab.h" #include <stdlib.h> -#include <string.h> +#include <math.h> #include <cmath> -#include <map> -#include <sstream> -#include <string> +#include <fenv.h> +#include "cuda-math.h" #include "../abstract_hardware_model.h" +#include "ptx_loader.h" +#include "cuda_device_printf.h" #include "../gpgpu-sim/gpu-sim.h" #include "../gpgpu-sim/shader.h" -#include "cuda-math.h" -#include "cuda_device_printf.h" -#include "ptx.tab.h" -#include "ptx_loader.h" +#include <assert.h> +#include <string.h> +#include <sstream> +#include <stdio.h> +#include <string> +#include <map> +#include <stdlib.h> -// Jin: include device runtime for CDP +//Jin: include device runtime for CDP #include "cuda_device_runtime.h" #include <stdarg.h> @@ -62,6425 +60,5629 @@ class ptx_recognizer; using half_float::half; + + const char *g_opcode_string[NUM_OPCODES] = { -#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) STR, -#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) STR, +#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR, +#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR, #include "opcodes.def" #undef OP_DEF #undef OP_W_DEF }; -// Using profiled information::check the TensorCoreMatrixArrangement.xls for -// details -unsigned thread_group_offset(int thread, unsigned wmma_type, - unsigned wmma_layout, unsigned type, int stride) { - unsigned offset; - unsigned load_a_row[8] = {0, 128, 0, 128, 64, 192, 64, 192}; - unsigned load_a_col[8] = {0, 8, 0, 8, 4, 12, 4, 12}; - unsigned load_b_row[8] = {0, 8, 0, 8, 4, 12, 4, 12}; - unsigned load_b_col[8] = {0, 128, 0, 128, 64, 192, 64, 192}; - unsigned load_c_float_row[8] = {0, 128, 8, 136, 64, 192, 72, 200}; - unsigned load_c_float_col[8] = {0, 8, 128, 136, 4, 12, 132, 140}; - unsigned load_c_half_row[8] = {0, 128, 8, 136, 64, 192, 72, 200}; - unsigned load_c_half_col[8] = {0, 8, 128, 136, 4, 12, 132, 140}; - unsigned thread_group = thread / 4; - unsigned in_tg_index = thread % 4; +//Using profiled information::check the TensorCoreMatrixArrangement.xls for details +unsigned thread_group_offset(int thread,unsigned wmma_type,unsigned wmma_layout,unsigned type,int stride){ - switch (wmma_type) { - case LOAD_A: - if (wmma_layout == ROW) - offset = load_a_row[thread_group] + 16 * in_tg_index; - else - offset = load_a_col[thread_group] + 16 * in_tg_index; - break; + unsigned offset; + unsigned load_a_row[8]={0,128,0,128,64,192,64,192}; + unsigned load_a_col[8]={0,8,0,8,4,12,4,12}; + unsigned load_b_row[8]={0,8,0,8,4,12,4,12}; + unsigned load_b_col[8]={0,128,0,128,64,192,64,192}; + unsigned load_c_float_row[8]={0,128,8,136,64,192,72,200}; + unsigned load_c_float_col[8]={0,8,128,136,4,12,132,140}; + unsigned load_c_half_row[8]={0,128,8,136,64,192,72,200}; + unsigned load_c_half_col[8]={0,8,128,136,4,12,132,140}; + unsigned thread_group = thread/4; + unsigned in_tg_index = thread%4; - case LOAD_B: - if (wmma_layout == ROW) - offset = load_b_row[thread_group] + 16 * in_tg_index; - else - offset = load_b_col[thread_group] + 16 * in_tg_index; - break; + switch(wmma_type){ + case LOAD_A: + if(wmma_layout==ROW) + offset=load_a_row[thread_group]+16*in_tg_index; + else + offset=load_a_col[thread_group]+16*in_tg_index; + break; - case LOAD_C: - case STORE_D: - if (type == F16_TYPE) { - if (wmma_layout == ROW) - offset = load_c_half_row[thread_group] + 16 * in_tg_index; - else - offset = load_c_half_col[thread_group] + in_tg_index; - } else { - if (wmma_layout == ROW) - offset = load_c_float_row[thread_group]; - else - offset = load_c_float_col[thread_group]; - switch (in_tg_index) { - case 0: - break; - case 1: - if (wmma_layout == ROW) - offset += 16; - else - offset += 1; - break; - case 2: - if (wmma_layout == ROW) - offset += 2; - else - offset += 32; - break; - case 3: - if (wmma_layout == ROW) - offset += 18; - else - offset += 33; - break; - default: - abort(); - } - } - break; + case LOAD_B: + if(wmma_layout==ROW) + offset=load_b_row[thread_group]+16*in_tg_index; + else + offset=load_b_col[thread_group]+16*in_tg_index; + break; - default: - abort(); - } - offset = (offset / 16) * stride + offset % 16; - return offset; + case LOAD_C: + case STORE_D: + if(type==F16_TYPE){ + if(wmma_layout==ROW) + offset=load_c_half_row[thread_group]+16*in_tg_index; + else + offset=load_c_half_col[thread_group]+in_tg_index; + } + else{ + if(wmma_layout==ROW) + offset=load_c_float_row[thread_group]; + else + offset=load_c_float_col[thread_group]; + + switch(in_tg_index){ + case 0: + break; + case 1: + if(wmma_layout==ROW) + offset+=16; + else + offset+=1; + break; + case 2: + if(wmma_layout==ROW) + offset+=2; + else + offset+=32; + break; + case 3: + if(wmma_layout==ROW) + offset+=18; + else + offset+=33; + break; + default: + abort(); + } + } + break; + + default: + abort(); + + } + offset = (offset/16)*stride+offset%16; + return offset; } -int acc_float_offset(int index, int wmma_layout, int stride) { - int c_row_offset[] = {0, 1, 32, 33, 4, 5, 36, 37}; - int c_col_offset[] = {0, 16, 2, 18, 64, 80, 66, 82}; - int offset; +int acc_float_offset(int index,int wmma_layout,int stride){ - if (wmma_layout == ROW) - offset = c_row_offset[index]; - else if (wmma_layout == COL) - offset = c_col_offset[index]; - else { - printf("wrong layout"); - abort(); - } - offset = (offset / 16) * stride + offset % 16; - return offset; + int c_row_offset[]={0,1,32,33,4,5,36,37}; + int c_col_offset[]={0,16,2,18,64,80,66,82}; + int offset; + + + if(wmma_layout==ROW) + offset=c_row_offset[index]; + else if(wmma_layout==COL) + offset=c_col_offset[index]; + else{ + printf("wrong layout"); + abort(); + } + offset = (offset/16)*stride+offset%16; + return offset; } -void inst_not_implemented(const ptx_instruction *pI); -ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, - operand_info dstInfo, unsigned type, - ptx_thread_info *thread); +void inst_not_implemented( const ptx_instruction * pI ) ; +ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread); -void sign_extend(ptx_reg_t &data, unsigned src_size, const operand_info &dst); +void sign_extend( ptx_reg_t &data, unsigned src_size, const operand_info &dst ); -void ptx_thread_info::set_reg(const symbol *reg, const ptx_reg_t &value) { - assert(reg != NULL); - if (reg->name() == "_") return; - assert(!m_regs.empty()); - assert(reg->uid() > 0); - m_regs.back()[reg] = value; - if (m_enable_debug_trace) m_debug_trace_regs_modified.back()[reg] = value; - m_last_set_operand_value = value; +void ptx_thread_info::set_reg( const symbol *reg, const ptx_reg_t &value ) +{ + assert( reg != NULL ); + if( reg->name() == "_" ) return; + assert( !m_regs.empty() ); + assert( reg->uid() > 0 ); + m_regs.back()[ reg ] = value; + if (m_enable_debug_trace ) + m_debug_trace_regs_modified.back()[ reg ] = value; + m_last_set_operand_value = value; } -void ptx_thread_info::print_reg_thread(char *fname) { - FILE *fp = fopen(fname, "w"); - assert(fp != NULL); +void ptx_thread_info::print_reg_thread(char * fname) +{ - int size = m_regs.size(); - - if (size > 0) { - reg_map_t reg = m_regs.back(); + FILE *fp= fopen(fname,"w"); + assert(fp!=NULL); - reg_map_t::const_iterator it; - for (it = reg.begin(); it != reg.end(); ++it) { - const std::string &name = it->first->name(); - const std::string &dec = it->first->decl_location(); - unsigned size = it->first->get_size_in_bytes(); - fprintf(fp, "%s %llu %s %d\n", name.c_str(), it->second, dec.c_str(), - size); - } - // m_regs.pop_back(); + int size = m_regs.size(); + + if(size>0) + { + reg_map_t reg = m_regs.back(); + + reg_map_t::const_iterator it; + for (it = reg.begin(); it != reg.end(); ++it) + { + const std::string &name = it->first->name(); + const std::string &dec= it->first->decl_location(); + unsigned size = it->first->get_size_in_bytes(); + fprintf(fp,"%s %llu %s %d\n", name.c_str(), it->second, dec.c_str(), size); + + } + //m_regs.pop_back(); } fclose(fp); -} -void ptx_thread_info::resume_reg_thread(char *fname, symbol_table *symtab) { - FILE *fp2 = fopen(fname, "r"); - assert(fp2 != NULL); - // m_regs.push_back( reg_map_t() ); - char line[200]; - while (fgets(line, sizeof line, fp2) != NULL) { - symbol *reg; - char *pch; - pch = strtok(line, " "); - char *name = pch; - reg = symtab->lookup(name); - ptx_reg_t data; - pch = strtok(NULL, " "); - data = atoi(pch); - pch = strtok(NULL, " "); - pch = strtok(NULL, " "); - m_regs.back()[reg] = data; } - fclose(fp2); + +void ptx_thread_info::resume_reg_thread(char * fname, symbol_table * symtab) +{ + + + FILE * fp2 = fopen(fname, "r"); + assert(fp2!=NULL); + //m_regs.push_back( reg_map_t() ); + char line [ 200 ]; + while ( fgets ( line, sizeof line, fp2 ) != NULL ) + { + symbol *reg; + char * pch; + pch = strtok (line," "); + char * name =pch; + reg= symtab->lookup(name); + ptx_reg_t data; + pch = strtok (NULL," "); + data = atoi(pch); + pch = strtok (NULL," "); + pch = strtok (NULL," "); + m_regs.back()[reg] = data; + } + fclose ( fp2 ); } + -ptx_reg_t ptx_thread_info::get_reg(const symbol *reg) { - static bool unfound_register_warned = false; - assert(reg != NULL); - assert(!m_regs.empty()); - reg_map_t::iterator regs_iter = m_regs.back().find(reg); - if (regs_iter == m_regs.back().end()) { - assert(reg->type()->get_key().is_reg()); - const std::string &name = reg->name(); - unsigned call_uid = m_callstack.back().m_call_uid; - ptx_reg_t uninit_reg; - uninit_reg.u32 = 0x0; - set_reg(reg, uninit_reg); // give it a value since we are going to warn the - // user anyway - std::string file_loc = get_location(); - if (!unfound_register_warned) { - printf( - "GPGPU-Sim PTX: WARNING (%s) ** reading undefined register \'%s\' " - "(cuid:%u). Setting to 0X00000000. This is okay if you are " - "simulating the native ISA" - "\n", - file_loc.c_str(), name.c_str(), call_uid); - unfound_register_warned = true; - } - regs_iter = m_regs.back().find(reg); - } - if (m_enable_debug_trace) - m_debug_trace_regs_read.back()[reg] = regs_iter->second; - return regs_iter->second; +ptx_reg_t ptx_thread_info::get_reg( const symbol *reg ) +{ + static bool unfound_register_warned = false; + assert( reg != NULL ); + assert( !m_regs.empty() ); + reg_map_t::iterator regs_iter = m_regs.back().find(reg); + if (regs_iter == m_regs.back().end()) { + assert( reg->type()->get_key().is_reg() ); + const std::string &name = reg->name(); + unsigned call_uid = m_callstack.back().m_call_uid; + ptx_reg_t uninit_reg; + uninit_reg.u32 = 0x0; + set_reg(reg, uninit_reg); // give it a value since we are going to warn the user anyway + std::string file_loc = get_location(); + if( !unfound_register_warned ) { + printf("GPGPU-Sim PTX: WARNING (%s) ** reading undefined register \'%s\' (cuid:%u). Setting to 0X00000000. This is okay if you are simulating the native ISA" + "\n", + file_loc.c_str(), name.c_str(), call_uid ); + unfound_register_warned = true; + } + regs_iter = m_regs.back().find(reg); + } + if (m_enable_debug_trace ) + m_debug_trace_regs_read.back()[ reg ] = regs_iter->second; + return regs_iter->second; } -ptx_reg_t ptx_thread_info::get_operand_value(const operand_info &op, - operand_info dstInfo, - unsigned opType, - ptx_thread_info *thread, - int derefFlag) { - ptx_reg_t result, tmp; +ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_info dstInfo, unsigned opType, ptx_thread_info *thread, int derefFlag ) +{ + ptx_reg_t result, tmp; - if (op.get_double_operand_type() == 0) { - if (((opType != BB128_TYPE) && (opType != BB64_TYPE) && - (opType != FF64_TYPE)) || - (op.get_addr_space() != undefined_space)) { - if (op.is_reg()) { - result = get_reg(op.get_symbol()); - } else if (op.is_builtin()) { - result.u32 = get_builtin(op.get_int(), op.get_addr_offset()); - } else if (op.is_immediate_address()) { - result.u64 = op.get_addr_offset(); - } else if (op.is_memory_operand()) { - // a few options here... - const symbol *sym = op.get_symbol(); - const type_info *type = sym->type(); - const type_info_key &info = type->get_key(); - if (info.is_reg()) { - const symbol *name = op.get_symbol(); - result.u64 = get_reg(name).u64 + op.get_addr_offset(); - } else if (info.is_param_kernel()) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if (info.is_param_local()) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if (info.is_global()) { - assert(op.get_addr_offset() == 0); - result.u64 = sym->get_address(); - } else if (info.is_local()) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if (info.is_const()) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if (op.is_shared()) { - result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); - } else if (op.is_sstarr()) { - result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); - } else { - const char *name = op.name().c_str(); - printf( - "GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory " - "operand type for %s\n", - name); - abort(); - } + if(op.get_double_operand_type() == 0) { + if(((opType != BB128_TYPE) && (opType != BB64_TYPE) && (opType != FF64_TYPE)) || (op.get_addr_space() != undefined_space)) { + if ( op.is_reg() ) { + result = get_reg( op.get_symbol() ); + } else if ( op.is_builtin()) { + result.u32 = get_builtin( op.get_int(), op.get_addr_offset() ); + } else if(op.is_immediate_address()){ + result.u64 = op.get_addr_offset(); + } else if ( op.is_memory_operand() ) { + // a few options here... + const symbol *sym = op.get_symbol(); + const type_info *type = sym->type(); + const type_info_key &info = type->get_key(); + + if ( info.is_reg() ) { + const symbol *name = op.get_symbol(); + result.u64 = get_reg(name).u64 + op.get_addr_offset(); + } else if ( info.is_param_kernel() ) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if ( info.is_param_local() ) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if ( info.is_global() ) { + assert( op.get_addr_offset() == 0 ); + result.u64 = sym->get_address(); + } else if ( info.is_local() ) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if ( info.is_const() ) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if ( op.is_shared() ) { + result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); + } else if ( op.is_sstarr() ) { + result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); + } else { + const char *name = op.name().c_str(); + printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory operand type for %s\n", name ); + abort(); + } + + } else if ( op.is_literal() ) { + result = op.get_literal_value(); + } else if ( op.is_label() ) { + result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_shared() ) { + result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_sstarr() ) { + result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_const() ) { + result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_global() ) { + result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_local() ) { + result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_function_address() ) { + result.u64 = (size_t)op.get_symbol()->get_pc(); + } else if ( op.is_param_kernel()) { + result.u64 = op.get_symbol()->get_address(); + }else { + const char *name = op.name().c_str(); + const symbol *sym2 = op.get_symbol(); + const type_info *type2 = sym2->type(); + const type_info_key &info2 = type2->get_key(); + if ( info2.is_param_kernel() ) { + result.u64 = sym2->get_address()+ op.get_addr_offset(); + } + else{ + printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand type for %s\n", name ); + assert(0); + } + } - } else if (op.is_literal()) { - result = op.get_literal_value(); - } else if (op.is_label()) { - result.u64 = op.get_symbol()->get_address(); - } else if (op.is_shared()) { - result.u64 = op.get_symbol()->get_address(); - } else if (op.is_sstarr()) { - result.u64 = op.get_symbol()->get_address(); - } else if (op.is_const()) { - result.u64 = op.get_symbol()->get_address(); - } else if (op.is_global()) { - result.u64 = op.get_symbol()->get_address(); - } else if (op.is_local()) { - result.u64 = op.get_symbol()->get_address(); - } else if (op.is_function_address()) { - result.u64 = (size_t)op.get_symbol()->get_pc(); - } else if (op.is_param_kernel()) { - result.u64 = op.get_symbol()->get_address(); + if(op.get_operand_lohi() == 1) + result.u64 = result.u64 & 0xFFFF; + else if(op.get_operand_lohi() == 2) + result.u64 = (result.u64>>16) & 0xFFFF; + } else if (opType == BB128_TYPE) { + // b128 + result.u128.lowest = get_reg( op.vec_symbol(0) ).u32; + result.u128.low = get_reg( op.vec_symbol(1) ).u32; + result.u128.high = get_reg( op.vec_symbol(2) ).u32; + result.u128.highest = get_reg( op.vec_symbol(3) ).u32; } else { - const char *name = op.name().c_str(); - const symbol *sym2 = op.get_symbol(); - const type_info *type2 = sym2->type(); - const type_info_key &info2 = type2->get_key(); - if (info2.is_param_kernel()) { - result.u64 = sym2->get_address() + op.get_addr_offset(); - } else { - printf( - "GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand " - "type for %s\n", - name); - assert(0); - } + // bb64 or ff64 + result.bits.ls = get_reg( op.vec_symbol(0) ).u32; + result.bits.ms = get_reg( op.vec_symbol(1) ).u32; } + } else if (op.get_double_operand_type() == 1) { + ptx_reg_t firstHalf, secondHalf; + firstHalf.u64 = get_reg( op.vec_symbol(0) ).u64; + secondHalf.u64 = get_reg( op.vec_symbol(1) ).u64; + if(op.get_operand_lohi() == 1) + secondHalf.u64 = secondHalf.u64 & 0xFFFF; + else if(op.get_operand_lohi() == 2) + secondHalf.u64 = (secondHalf.u64>>16) & 0xFFFF; + result.u64 = firstHalf.u64 + secondHalf.u64; + } else if (op.get_double_operand_type() == 2) { + // s[reg1 += reg2] + // reg1 is incremented after value is returned: the value returned is s[reg1] + ptx_reg_t firstHalf, secondHalf; + firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; + secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; + if(op.get_operand_lohi() == 1) + secondHalf.u64 = secondHalf.u64 & 0xFFFF; + else if(op.get_operand_lohi() == 2) + secondHalf.u64 = (secondHalf.u64>>16) & 0xFFFF; + result.u64 = firstHalf.u64; + firstHalf.u64 = firstHalf.u64 + secondHalf.u64; + set_reg(op.vec_symbol(0),firstHalf); + } else if (op.get_double_operand_type() == 3) { + // s[reg += immediate] + // reg is incremented after value is returned: the value returned is s[reg] + ptx_reg_t firstHalf; + firstHalf.u64 = get_reg(op.get_symbol()).u64; + result.u64 = firstHalf.u64; + firstHalf.u64 = firstHalf.u64 + op.get_addr_offset(); + set_reg(op.get_symbol(),firstHalf); + } - if (op.get_operand_lohi() == 1) - result.u64 = result.u64 & 0xFFFF; - else if (op.get_operand_lohi() == 2) - result.u64 = (result.u64 >> 16) & 0xFFFF; - } else if (opType == BB128_TYPE) { - // b128 - result.u128.lowest = get_reg(op.vec_symbol(0)).u32; - result.u128.low = get_reg(op.vec_symbol(1)).u32; - result.u128.high = get_reg(op.vec_symbol(2)).u32; - result.u128.highest = get_reg(op.vec_symbol(3)).u32; - } else { - // bb64 or ff64 - result.bits.ls = get_reg(op.vec_symbol(0)).u32; - result.bits.ms = get_reg(op.vec_symbol(1)).u32; - } - } else if (op.get_double_operand_type() == 1) { - ptx_reg_t firstHalf, secondHalf; - firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; - secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; - if (op.get_operand_lohi() == 1) - secondHalf.u64 = secondHalf.u64 & 0xFFFF; - else if (op.get_operand_lohi() == 2) - secondHalf.u64 = (secondHalf.u64 >> 16) & 0xFFFF; - result.u64 = firstHalf.u64 + secondHalf.u64; - } else if (op.get_double_operand_type() == 2) { - // s[reg1 += reg2] - // reg1 is incremented after value is returned: the value returned is - // s[reg1] - ptx_reg_t firstHalf, secondHalf; - firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; - secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; - if (op.get_operand_lohi() == 1) - secondHalf.u64 = secondHalf.u64 & 0xFFFF; - else if (op.get_operand_lohi() == 2) - secondHalf.u64 = (secondHalf.u64 >> 16) & 0xFFFF; - result.u64 = firstHalf.u64; - firstHalf.u64 = firstHalf.u64 + secondHalf.u64; - set_reg(op.vec_symbol(0), firstHalf); - } else if (op.get_double_operand_type() == 3) { - // s[reg += immediate] - // reg is incremented after value is returned: the value returned is s[reg] - ptx_reg_t firstHalf; - firstHalf.u64 = get_reg(op.get_symbol()).u64; - result.u64 = firstHalf.u64; - firstHalf.u64 = firstHalf.u64 + op.get_addr_offset(); - set_reg(op.get_symbol(), firstHalf); - } - - ptx_reg_t finalResult; - memory_space *mem = NULL; - size_t size = 0; - int t = 0; - finalResult.u64 = 0; + ptx_reg_t finalResult; + memory_space *mem = NULL; + size_t size=0; + int t=0; + finalResult.u64=0; - // complete other cases for reading from memory, such as reading from other - // const memory - if ((op.get_addr_space() == global_space) && (derefFlag)) { - // global memory - g[4], g[$r0] - mem = thread->get_global_memory(); - type_info_key::type_decode(opType, size, t); - mem->read(result.u32, size / 8, &finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = global_space; + //complete other cases for reading from memory, such as reading from other const memory + if((op.get_addr_space() == global_space)&&(derefFlag)) { + // global memory - g[4], g[$r0] + mem = thread->get_global_memory(); + type_info_key::type_decode(opType,size,t); + mem->read(result.u32,size/8,&finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = global_space; - if (opType == S16_TYPE || opType == S32_TYPE) - sign_extend(finalResult, size, dstInfo); - } else if ((op.get_addr_space() == shared_space) && (derefFlag)) { - // shared memory - s[4], s[$r0] - mem = thread->m_shared_mem; - type_info_key::type_decode(opType, size, t); - mem->read(result.u32, size / 8, &finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = shared_space; + if( opType == S16_TYPE || opType == S32_TYPE ) + sign_extend(finalResult,size,dstInfo); + } else if((op.get_addr_space() == shared_space)&&(derefFlag)) { + // shared memory - s[4], s[$r0] + mem = thread->m_shared_mem; + type_info_key::type_decode(opType,size,t); + mem->read(result.u32,size/8,&finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = shared_space; - if (opType == S16_TYPE || opType == S32_TYPE) - sign_extend(finalResult, size, dstInfo); - } else if ((op.get_addr_space() == const_space) && (derefFlag)) { - // const memory - ce0c1[4], ce0c1[$r0] - mem = thread->get_global_memory(); - type_info_key::type_decode(opType, size, t); - mem->read((result.u32 + op.get_const_mem_offset()), size / 8, - &finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = const_space; - if (opType == S16_TYPE || opType == S32_TYPE) - sign_extend(finalResult, size, dstInfo); - } else if ((op.get_addr_space() == local_space) && (derefFlag)) { - // local memory - l0[4], l0[$r0] - mem = thread->m_local_mem; - type_info_key::type_decode(opType, size, t); - mem->read(result.u32, size / 8, &finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = local_space; - if (opType == S16_TYPE || opType == S32_TYPE) - sign_extend(finalResult, size, dstInfo); - } else { - finalResult = result; - } + if( opType == S16_TYPE || opType == S32_TYPE ) + sign_extend(finalResult,size,dstInfo); + } else if((op.get_addr_space() == const_space)&&(derefFlag)) { + // const memory - ce0c1[4], ce0c1[$r0] + mem = thread->get_global_memory(); + type_info_key::type_decode(opType,size,t); + mem->read((result.u32 + op.get_const_mem_offset()),size/8,&finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = const_space; + if( opType == S16_TYPE || opType == S32_TYPE ) + sign_extend(finalResult,size,dstInfo); + } else if((op.get_addr_space() == local_space)&&(derefFlag)) { + // local memory - l0[4], l0[$r0] + mem = thread->m_local_mem; + type_info_key::type_decode(opType,size,t); + mem->read(result.u32,size/8,&finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = local_space; + if( opType == S16_TYPE || opType == S32_TYPE ) + sign_extend(finalResult,size,dstInfo); + } else { + finalResult = result; + } - if ((op.get_operand_neg() == true) && (derefFlag)) { - switch (opType) { + if((op.get_operand_neg() == true)&&(derefFlag)) { + switch( opType ) { // Default to f32 for now, need to add support for others case S8_TYPE: case U8_TYPE: case B8_TYPE: - finalResult.s8 = -finalResult.s8; - break; + finalResult.s8 = -finalResult.s8; + break; case S16_TYPE: case U16_TYPE: case B16_TYPE: - finalResult.s16 = -finalResult.s16; - break; + finalResult.s16 = -finalResult.s16; + break; case S32_TYPE: case U32_TYPE: case B32_TYPE: - finalResult.s32 = -finalResult.s32; - break; + finalResult.s32 = -finalResult.s32; + break; case S64_TYPE: case U64_TYPE: case B64_TYPE: - finalResult.s64 = -finalResult.s64; - break; + finalResult.s64 = -finalResult.s64; + break; case F16_TYPE: - finalResult.f16 = -finalResult.f16; - break; + finalResult.f16 = -finalResult.f16; + break; case F32_TYPE: - finalResult.f32 = -finalResult.f32; - break; + finalResult.f32 = -finalResult.f32; + break; case F64_TYPE: case FF64_TYPE: - finalResult.f64 = -finalResult.f64; - break; + finalResult.f64 = -finalResult.f64; + break; default: - assert(0); - } - } + assert(0); + } + + } + + return finalResult; - return finalResult; } -unsigned get_operand_nbits(const operand_info &op) { - if (op.is_reg()) { - const symbol *sym = op.get_symbol(); - const type_info *typ = sym->type(); - type_info_key t = typ->get_key(); - switch (t.scalar_type()) { - case PRED_TYPE: - return 1; - case B8_TYPE: - case S8_TYPE: - case U8_TYPE: - return 8; - case S16_TYPE: - case U16_TYPE: - case F16_TYPE: - case B16_TYPE: - return 16; - case S32_TYPE: - case U32_TYPE: - case F32_TYPE: - case B32_TYPE: - return 32; - case S64_TYPE: - case U64_TYPE: - case F64_TYPE: - case B64_TYPE: - return 64; +unsigned get_operand_nbits( const operand_info &op ) +{ + if ( op.is_reg() ) { + const symbol *sym = op.get_symbol(); + const type_info *typ = sym->type(); + type_info_key t = typ->get_key(); + switch( t.scalar_type() ) { + case PRED_TYPE: + return 1; + case B8_TYPE: case S8_TYPE: case U8_TYPE: + return 8; + case S16_TYPE: case U16_TYPE: case F16_TYPE: case B16_TYPE: + return 16; + case S32_TYPE: case U32_TYPE: case F32_TYPE: case B32_TYPE: + return 32; + case S64_TYPE: case U64_TYPE: case F64_TYPE: case B64_TYPE: + return 64; default: - printf("ERROR: unknown register type\n"); - fflush(stdout); - abort(); - } - } else { - printf( - "ERROR: Need to implement get_operand_nbits() for currently " - "unsupported operand_info type\n"); - fflush(stdout); - abort(); - } + printf("ERROR: unknown register type\n"); + fflush(stdout); + abort(); + } + } else { + printf("ERROR: Need to implement get_operand_nbits() for currently unsupported operand_info type\n"); + fflush(stdout); + abort(); + } - return 0; + return 0; } -void ptx_thread_info::get_vector_operand_values(const operand_info &op, - ptx_reg_t *ptx_regs, - unsigned num_elements) { - assert(op.is_vector()); - assert(num_elements <= 8); +void ptx_thread_info::get_vector_operand_values( const operand_info &op, ptx_reg_t* ptx_regs, unsigned num_elements ) +{ + assert( op.is_vector() ); + assert( num_elements <= 8 ); - for (int idx = num_elements - 1; idx >= 0; --idx) { - const symbol *sym = NULL; - sym = op.vec_symbol(idx); - if (strcmp(sym->name().c_str(), "_") != 0) { - reg_map_t::iterator reg_iter = m_regs.back().find(sym); - assert(reg_iter != m_regs.back().end()); - ptx_regs[idx] = reg_iter->second; - } - } + for (int idx = num_elements - 1; idx >= 0; --idx) { + const symbol *sym = NULL; + sym = op.vec_symbol(idx); + if( strcmp(sym->name().c_str(),"_") != 0) { + reg_map_t::iterator reg_iter = m_regs.back().find(sym); + assert( reg_iter != m_regs.back().end() ); + ptx_regs[idx] = reg_iter->second; + } + } } -void sign_extend(ptx_reg_t &data, unsigned src_size, const operand_info &dst) { - if (!dst.is_reg()) return; - unsigned dst_size = get_operand_nbits(dst); - if (src_size >= dst_size) return; - // src_size < dst_size - unsigned long long mask = 1; - mask <<= (src_size - 1); - if ((mask & data.u64) == 0) { - // no need to sign extend - return; - } - // need to sign extend - mask = 1; - mask <<= dst_size - src_size; - mask -= 1; - mask <<= src_size; - data.u64 |= mask; +void sign_extend( ptx_reg_t &data, unsigned src_size, const operand_info &dst ) +{ + if( !dst.is_reg() ) + return; + unsigned dst_size = get_operand_nbits( dst ); + if( src_size >= dst_size ) + return; + // src_size < dst_size + unsigned long long mask = 1; + mask <<= (src_size-1); + if( (mask & data.u64) == 0 ) { + // no need to sign extend + return; + } + // need to sign extend + mask = 1; + mask <<= dst_size-src_size; + mask -= 1; + mask <<= src_size; + data.u64 |= mask; } -void ptx_thread_info::set_operand_value(const operand_info &dst, - const ptx_reg_t &data, unsigned type, - ptx_thread_info *thread, - const ptx_instruction *pI, int overflow, - int carry) { - thread->set_operand_value(dst, data, type, thread, pI); +void ptx_thread_info::set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI, int overflow, int carry ) +{ + thread->set_operand_value( dst, data, type, thread, pI ); - if (dst.get_double_operand_type() == -2) { - ptx_reg_t predValue; + if (dst.get_double_operand_type() == -2) + { + ptx_reg_t predValue; + + const symbol *sym = dst.vec_symbol(0); + predValue.u64 = (m_regs.back()[ sym ].u64) & ~(0x0C); + predValue.u64 |= ((overflow & 0x01)<<3); + predValue.u64 |= ((carry & 0x01)<<2); - const symbol *sym = dst.vec_symbol(0); - predValue.u64 = (m_regs.back()[sym].u64) & ~(0x0C); - predValue.u64 |= ((overflow & 0x01) << 3); - predValue.u64 |= ((carry & 0x01) << 2); + set_reg(sym,predValue); + } + else if (dst.get_double_operand_type() == 0) + { + //intentionally do nothing + } + else + { + printf("Unexpected double destination\n"); + assert(0); + } - set_reg(sym, predValue); - } else if (dst.get_double_operand_type() == 0) { - // intentionally do nothing - } else { - printf("Unexpected double destination\n"); - assert(0); - } } -void ptx_thread_info::set_operand_value(const operand_info &dst, - const ptx_reg_t &data, unsigned type, - ptx_thread_info *thread, - const ptx_instruction *pI) { - ptx_reg_t dstData; - memory_space *mem = NULL; - size_t size; - int t; +void ptx_thread_info::set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI ) +{ + ptx_reg_t dstData; + memory_space *mem = NULL; + size_t size; + int t; - type_info_key::type_decode(type, size, t); + type_info_key::type_decode(type,size,t); - /*complete this section for other cases*/ - if (dst.get_addr_space() == undefined_space) { - ptx_reg_t setValue; - setValue.u64 = data.u64; + /*complete this section for other cases*/ + if(dst.get_addr_space() == undefined_space) + { + ptx_reg_t setValue; + setValue.u64 = data.u64; - // Double destination in set instruction ($p0|$p1) - second is negation of - // first - if (dst.get_double_operand_type() == -1) { - ptx_reg_t setValue2; - const symbol *name1 = dst.vec_symbol(0); - const symbol *name2 = dst.vec_symbol(1); + // Double destination in set instruction ($p0|$p1) - second is negation of first + if (dst.get_double_operand_type() == -1) + { + ptx_reg_t setValue2; + const symbol *name1 = dst.vec_symbol(0); + const symbol *name2 = dst.vec_symbol(1); - if ((type == F16_TYPE) || (type == F32_TYPE) || (type == F64_TYPE) || - (type == FF64_TYPE)) { - setValue2.f32 = (setValue.u64 == 0) ? 1.0f : 0.0f; - } else { - setValue2.u32 = (setValue.u64 == 0) ? 0xFFFFFFFF : 0; + if ( (type==F16_TYPE)||(type==F32_TYPE)||(type==F64_TYPE)||(type==FF64_TYPE) ) { + setValue2.f32 = (setValue.u64==0)?1.0f:0.0f; + } else { + setValue2.u32 = (setValue.u64==0)?0xFFFFFFFF:0; + } + + set_reg(name1,setValue); + set_reg(name2,setValue2); } - set_reg(name1, setValue); - set_reg(name2, setValue2); - } + // Double destination in cvt,shr,mul,etc. instruction ($p0|$r4) - second register operand receives data, first predicate operand + // is set as $p0=($r4!=0) + // Also for Double destination in set instruction ($p0/$r1) + else if ((dst.get_double_operand_type() == -2)||(dst.get_double_operand_type() == -3)) + { + ptx_reg_t predValue; + const symbol *predName = dst.vec_symbol(0); + const symbol *regName = dst.vec_symbol(1); + predValue.u64 = 0; - // Double destination in cvt,shr,mul,etc. instruction ($p0|$r4) - second - // register operand receives data, first predicate operand - // is set as $p0=($r4!=0) - // Also for Double destination in set instruction ($p0/$r1) - else if ((dst.get_double_operand_type() == -2) || - (dst.get_double_operand_type() == -3)) { - ptx_reg_t predValue; - const symbol *predName = dst.vec_symbol(0); - const symbol *regName = dst.vec_symbol(1); - predValue.u64 = 0; + switch ( type ) { + case S8_TYPE: + if((setValue.s8 & 0x7F) == 0) + predValue.u64 |= 1; + break; + case S16_TYPE: + if((setValue.s16 & 0x7FFF) == 0) + predValue.u64 |= 1; + break; + case S32_TYPE: + if((setValue.s32 & 0x7FFFFFFF) == 0) + predValue.u64 |= 1; + break; + case S64_TYPE: + if((setValue.s64 & 0x7FFFFFFFFFFFFFFF) == 0) + predValue.u64 |= 1; + break; + case U8_TYPE: + case B8_TYPE: + if(setValue.u8 == 0) + predValue.u64 |= 1; + break; + case U16_TYPE: + case B16_TYPE: + if(setValue.u16 == 0) + predValue.u64 |= 1; + break; + case U32_TYPE: + case B32_TYPE: + if(setValue.u32 == 0) + predValue.u64 |= 1; + break; + case U64_TYPE: + case B64_TYPE: + if(setValue.u64 == 0) + predValue.u64 |= 1; + break; + case F16_TYPE: + if(setValue.f16 == 0) + predValue.u64 |= 1; + break; + case F32_TYPE: + if(setValue.f32 == 0) + predValue.u64 |= 1; + break; + case F64_TYPE: + case FF64_TYPE: + if(setValue.f64 == 0) + predValue.u64 |= 1; + break; + default: assert(0); break; + } - switch (type) { - case S8_TYPE: - if ((setValue.s8 & 0x7F) == 0) predValue.u64 |= 1; - break; - case S16_TYPE: - if ((setValue.s16 & 0x7FFF) == 0) predValue.u64 |= 1; - break; - case S32_TYPE: - if ((setValue.s32 & 0x7FFFFFFF) == 0) predValue.u64 |= 1; - break; - case S64_TYPE: - if ((setValue.s64 & 0x7FFFFFFFFFFFFFFF) == 0) predValue.u64 |= 1; - break; - case U8_TYPE: - case B8_TYPE: - if (setValue.u8 == 0) predValue.u64 |= 1; - break; - case U16_TYPE: - case B16_TYPE: - if (setValue.u16 == 0) predValue.u64 |= 1; - break; - case U32_TYPE: - case B32_TYPE: - if (setValue.u32 == 0) predValue.u64 |= 1; - break; - case U64_TYPE: - case B64_TYPE: - if (setValue.u64 == 0) predValue.u64 |= 1; - break; - case F16_TYPE: - if (setValue.f16 == 0) predValue.u64 |= 1; - break; - case F32_TYPE: - if (setValue.f32 == 0) predValue.u64 |= 1; - break; - case F64_TYPE: - case FF64_TYPE: - if (setValue.f64 == 0) predValue.u64 |= 1; - break; - default: - assert(0); - break; - } - if ((type == S8_TYPE) || (type == S16_TYPE) || (type == S32_TYPE) || - (type == S64_TYPE) || (type == U8_TYPE) || (type == U16_TYPE) || - (type == U32_TYPE) || (type == U64_TYPE) || (type == B8_TYPE) || - (type == B16_TYPE) || (type == B32_TYPE) || (type == B64_TYPE)) { - if ((setValue.u32 & (1 << (size - 1))) != 0) predValue.u64 |= 1 << 1; - } - if (type == F32_TYPE) { - if (setValue.f32 < 0) predValue.u64 |= 1 << 1; - } + if ( (type==S8_TYPE)||(type==S16_TYPE)||(type==S32_TYPE)||(type==S64_TYPE)|| + (type==U8_TYPE)||(type==U16_TYPE)||(type==U32_TYPE)||(type==U64_TYPE)|| + (type==B8_TYPE)||(type==B16_TYPE)||(type==B32_TYPE)||(type==B64_TYPE)) { + if((setValue.u32 & (1<<(size-1))) != 0) + predValue.u64 |= 1<<1; + } + if ( type==F32_TYPE ) { + if(setValue.f32 < 0) + predValue.u64 |= 1<<1; + } - if (dst.get_operand_lohi() == 1) { - setValue.u64 = - ((m_regs.back()[regName].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); - } else if (dst.get_operand_lohi() == 2) { - setValue.u64 = ((m_regs.back()[regName].u64) & (~(0xFFFF0000))) + - ((data.u64 << 16) & 0xFFFF0000); - } + if(dst.get_operand_lohi() == 1) + { + setValue.u64 = ((m_regs.back()[ regName ].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); + } + else if(dst.get_operand_lohi() == 2) + { + setValue.u64 = ((m_regs.back()[ regName ].u64) & (~(0xFFFF0000))) + ((data.u64<<16) & 0xFFFF0000); + } - set_reg(predName, predValue); - set_reg(regName, setValue); - } else if (type == BB128_TYPE) { - // b128 stuff here. - ptx_reg_t setValue2, setValue3, setValue4; - setValue.u64 = 0; - setValue2.u64 = 0; - setValue3.u64 = 0; - setValue4.u64 = 0; - setValue.u32 = data.u128.lowest; - setValue2.u32 = data.u128.low; - setValue3.u32 = data.u128.high; - setValue4.u32 = data.u128.highest; + set_reg(predName,predValue); + set_reg(regName,setValue); + } + else if (type == BB128_TYPE) + { + //b128 stuff here. + ptx_reg_t setValue2, setValue3, setValue4; + setValue.u64 = 0; + setValue2.u64 = 0; + setValue3.u64 = 0; + setValue4.u64 = 0; + setValue.u32 = data.u128.lowest; + setValue2.u32 = data.u128.low; + setValue3.u32 = data.u128.high; + setValue4.u32 = data.u128.highest; - const symbol *name1, *name2, *name3, *name4 = NULL; + const symbol *name1, *name2, *name3, *name4 = NULL; - name1 = dst.vec_symbol(0); - name2 = dst.vec_symbol(1); - name3 = dst.vec_symbol(2); - name4 = dst.vec_symbol(3); + name1 = dst.vec_symbol(0); + name2 = dst.vec_symbol(1); + name3 = dst.vec_symbol(2); + name4 = dst.vec_symbol(3); - set_reg(name1, setValue); - set_reg(name2, setValue2); - set_reg(name3, setValue3); - set_reg(name4, setValue4); - } else if (type == BB64_TYPE || type == FF64_TYPE) { - // ptxplus version of storing 64 bit values to registers stores to two - // adjacent registers - ptx_reg_t setValue2; - setValue.u32 = 0; - setValue2.u32 = 0; + set_reg(name1,setValue); + set_reg(name2,setValue2); + set_reg(name3,setValue3); + set_reg(name4,setValue4); + } + else if (type == BB64_TYPE || type == FF64_TYPE) + { + //ptxplus version of storing 64 bit values to registers stores to two adjacent registers + ptx_reg_t setValue2; + setValue.u32 = 0; + setValue2.u32 = 0; - setValue.u32 = data.bits.ls; - setValue2.u32 = data.bits.ms; + setValue.u32 = data.bits.ls; + setValue2.u32 = data.bits.ms; - const symbol *name1, *name2 = NULL; + const symbol *name1, *name2 = NULL; - name1 = dst.vec_symbol(0); - name2 = dst.vec_symbol(1); + name1 = dst.vec_symbol(0); + name2 = dst.vec_symbol(1); - set_reg(name1, setValue); - set_reg(name2, setValue2); - } else { - if (dst.get_operand_lohi() == 1) { - setValue.u64 = ((m_regs.back()[dst.get_symbol()].u64) & (~(0xFFFF))) + - (data.u64 & 0xFFFF); - } else if (dst.get_operand_lohi() == 2) { - setValue.u64 = - ((m_regs.back()[dst.get_symbol()].u64) & (~(0xFFFF0000))) + - ((data.u64 << 16) & 0xFFFF0000); + set_reg(name1,setValue); + set_reg(name2,setValue2); } - set_reg(dst.get_symbol(), setValue); - } - } + else + { + if(dst.get_operand_lohi() == 1) + { + setValue.u64 = ((m_regs.back()[ dst.get_symbol() ].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); + } + else if(dst.get_operand_lohi() == 2) + { + setValue.u64 = ((m_regs.back()[ dst.get_symbol() ].u64) & (~(0xFFFF0000))) + ((data.u64<<16) & 0xFFFF0000); + } + set_reg(dst.get_symbol(),setValue); + } + } - // global memory - g[4], g[$r0] - else if (dst.get_addr_space() == global_space) { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->get_global_memory(); - type_info_key::type_decode(type, size, t); + // global memory - g[4], g[$r0] + else if(dst.get_addr_space() == global_space) + { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->get_global_memory(); + type_info_key::type_decode(type,size,t); - mem->write(dstData.u32, size / 8, &data.u128, thread, pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = global_space; - } + mem->write(dstData.u32,size/8,&data.u128,thread,pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = global_space; + } - // shared memory - s[4], s[$r0] - else if (dst.get_addr_space() == shared_space) { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->m_shared_mem; - type_info_key::type_decode(type, size, t); + // shared memory - s[4], s[$r0] + else if(dst.get_addr_space() == shared_space) + { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->m_shared_mem; + type_info_key::type_decode(type,size,t); - mem->write(dstData.u32, size / 8, &data.u128, thread, pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = shared_space; - } + mem->write(dstData.u32,size/8,&data.u128,thread,pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = shared_space; + } - // local memory - l0[4], l0[$r0] - else if (dst.get_addr_space() == local_space) { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->m_local_mem; - type_info_key::type_decode(type, size, t); + // local memory - l0[4], l0[$r0] + else if(dst.get_addr_space() == local_space) + { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->m_local_mem; + type_info_key::type_decode(type,size,t); + + mem->write(dstData.u32,size/8,&data.u128,thread,pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = local_space; + } + + else + { + printf("Destination stores to unknown location."); + assert(0); + } - mem->write(dstData.u32, size / 8, &data.u128, thread, pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = local_space; - } - else { - printf("Destination stores to unknown location."); - assert(0); - } } -void ptx_thread_info::set_vector_operand_values(const operand_info &dst, - const ptx_reg_t &data1, - const ptx_reg_t &data2, - const ptx_reg_t &data3, - const ptx_reg_t &data4) { - unsigned num_elements = dst.get_vect_nelem(); - if (num_elements > 0) { - set_reg(dst.vec_symbol(0), data1); - if (num_elements > 1) { - set_reg(dst.vec_symbol(1), data2); - if (num_elements > 2) { - set_reg(dst.vec_symbol(2), data3); - if (num_elements > 3) { - set_reg(dst.vec_symbol(3), data4); - } - } - } - } +void ptx_thread_info::set_vector_operand_values( const operand_info &dst, + const ptx_reg_t &data1, + const ptx_reg_t &data2, + const ptx_reg_t &data3, + const ptx_reg_t &data4 ) +{ + unsigned num_elements = dst.get_vect_nelem(); + if (num_elements > 0) { + set_reg(dst.vec_symbol(0), data1); + if (num_elements > 1) { + set_reg(dst.vec_symbol(1), data2); + if (num_elements > 2) { + set_reg(dst.vec_symbol(2), data3); + if (num_elements > 3) { + set_reg(dst.vec_symbol(3), data4); + } + } + } + } - m_last_set_operand_value = data1; + m_last_set_operand_value = data1; } -void ptx_thread_info::set_wmma_vector_operand_values( - const operand_info &dst, const ptx_reg_t &data1, const ptx_reg_t &data2, - const ptx_reg_t &data3, const ptx_reg_t &data4, const ptx_reg_t &data5, - const ptx_reg_t &data6, const ptx_reg_t &data7, const ptx_reg_t &data8) { - unsigned num_elements = dst.get_vect_nelem(); - if (num_elements == 8) { - set_reg(dst.vec_symbol(0), data1); - set_reg(dst.vec_symbol(1), data2); - set_reg(dst.vec_symbol(2), data3); - set_reg(dst.vec_symbol(3), data4); - set_reg(dst.vec_symbol(4), data5); - set_reg(dst.vec_symbol(5), data6); - set_reg(dst.vec_symbol(6), data7); - set_reg(dst.vec_symbol(7), data8); - } else { - printf("error:set_wmma_vector_operands"); - } +void ptx_thread_info::set_wmma_vector_operand_values( const operand_info &dst, + const ptx_reg_t &data1, + const ptx_reg_t &data2, + const ptx_reg_t &data3, + const ptx_reg_t &data4, + const ptx_reg_t &data5, + const ptx_reg_t &data6, + const ptx_reg_t &data7, + const ptx_reg_t &data8 ) +{ + unsigned num_elements = dst.get_vect_nelem(); + if (num_elements == 8) { + set_reg(dst.vec_symbol(0), data1); + set_reg(dst.vec_symbol(1), data2); + set_reg(dst.vec_symbol(2), data3); + set_reg(dst.vec_symbol(3), data4); + set_reg(dst.vec_symbol(4), data5); + set_reg(dst.vec_symbol(5), data6); + set_reg(dst.vec_symbol(6), data7); + set_reg(dst.vec_symbol(7), data8); + } + else{ + printf("error:set_wmma_vector_operands"); + } - m_last_set_operand_value = data8; + m_last_set_operand_value = data8; } -#define my_abs(a) (((a) < 0) ? (-a) : (a)) +#define my_abs(a) (((a)<0)?(-a):(a)) -#define MY_MAX_I(a, b) (a > b) ? a : b -#define MY_MAX_F(a, b) isNaN(a) ? b : isNaN(b) ? a : (a > b) ? a : b +#define MY_MAX_I(a,b) (a > b) ? a : b +#define MY_MAX_F(a,b) isNaN(a) ? b : isNaN(b) ? a : (a > b) ? a : b -#define MY_MIN_I(a, b) (a < b) ? a : b -#define MY_MIN_F(a, b) isNaN(a) ? b : isNaN(b) ? a : (a < b) ? a : b +#define MY_MIN_I(a,b) (a < b) ? a : b +#define MY_MIN_F(a,b) isNaN(a) ? b : isNaN(b) ? a : (a < b) ? a : b -#define MY_INC_I(a, b) (a >= b) ? 0 : a + 1 -#define MY_DEC_I(a, b) ((a == 0) || (a > b)) ? b : a - 1 +#define MY_INC_I(a,b) (a >= b) ? 0 : a+1 +#define MY_DEC_I(a,b) ((a == 0) || (a > b)) ? b : a-1 -#define MY_CAS_I(a, b, c) (a == b) ? c : a +#define MY_CAS_I(a,b,c) (a == b) ? c : a -#define MY_EXCH(a, b) b +#define MY_EXCH(a,b) b -void abs_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void abs_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch (i_type) { - case S16_TYPE: - d.s16 = my_abs(a.s16); - break; - case S32_TYPE: - d.s32 = my_abs(a.s32); - break; - case S64_TYPE: - d.s64 = my_abs(a.s64); - break; - case U16_TYPE: - d.s16 = my_abs(a.u16); - break; - case U32_TYPE: - d.s32 = my_abs(a.u32); - break; - case U64_TYPE: - d.s64 = my_abs(a.u64); - break; - case F32_TYPE: - d.f32 = my_abs(a.f32); - break; - case F64_TYPE: - case FF64_TYPE: - d.f64 = my_abs(a.f64); - break; - default: + + switch ( i_type ) { + case S16_TYPE: d.s16 = my_abs(a.s16); break; + case S32_TYPE: d.s32 = my_abs(a.s32); break; + case S64_TYPE: d.s64 = my_abs(a.s64); break; + case U16_TYPE: d.s16 = my_abs(a.u16); break; + case U32_TYPE: d.s32 = my_abs(a.u32); break; + case U64_TYPE: d.s64 = my_abs(a.u64); break; + case F32_TYPE: d.f32 = my_abs(a.f32); break; + case F64_TYPE: case FF64_TYPE: d.f64 = my_abs(a.f64); break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -void addp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - // PTXPlus add instruction with carry (carry is kept in a predicate) register - ptx_reg_t src1_data, src2_data, src3_data, data; - int overflow = 0; - int carry = 0; +void addp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + //PTXPlus add instruction with carry (carry is kept in a predicate) register + ptx_reg_t src1_data, src2_data, src3_data, data; + int overflow = 0; + int carry = 0; - const operand_info &dst = - pI->dst(); // get operand info of sources and destination - const operand_info &src1 = - pI->src1(); // use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); + const operand_info &dst = pI->dst(); //get operand info of sources and destination + const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - unsigned rounding_mode = pI->rounding_mode(); - int orig_rm = fegetround(); - switch (rounding_mode) { - case RN_OPTION: - break; - case RZ_OPTION: - fesetround(FE_TOWARDZERO); - break; - default: - assert(0); - break; - } + unsigned rounding_mode = pI->rounding_mode(); + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } - // performs addition. Sets carry and overflow if needed. - // src3_data.pred&0x4 is the carry flag - switch (i_type) { - case S8_TYPE: - data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF) + - (src3_data.pred & 0x4); - if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) == 0) { - overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; - } - carry = (data.u64 & 0x000000100) >> 8; + //performs addition. Sets carry and overflow if needed. + //src3_data.pred&0x4 is the carry flag + switch ( i_type ) { + case S8_TYPE: + data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF) + (src3_data.pred & 0x4); + if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) == 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } + carry = (data.u64 & 0x000000100)>>8; break; - case S16_TYPE: - data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF) + - (src3_data.pred & 0x4); - if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) == 0) { - overflow = - ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; - } - carry = (data.u64 & 0x000010000) >> 16; + case S16_TYPE: + data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF) + (src3_data.pred & 0x4); + if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) == 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } + carry = (data.u64 & 0x000010000)>>16; break; - case S32_TYPE: - data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF) + - (src3_data.pred & 0x4); - if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) == 0) { - overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 - ? 0 - : 1; - } - carry = (data.u64 & 0x100000000) >> 32; + case S32_TYPE: + data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF) + (src3_data.pred & 0x4); + if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) == 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } + carry = (data.u64 & 0x100000000)>>32; break; - case S64_TYPE: + case S64_TYPE: data.s64 = src1_data.s64 + src2_data.s64 + (src3_data.pred & 0x4); break; - case U8_TYPE: - data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF) + - (src3_data.pred & 0x4); - carry = (data.u64 & 0x100) >> 8; + case U8_TYPE: + data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF) + (src3_data.pred & 0x4); + carry = (data.u64 & 0x100)>>8; break; - case U16_TYPE: - data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF) + - (src3_data.pred & 0x4); - carry = (data.u64 & 0x10000) >> 16; + case U16_TYPE: + data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF) + (src3_data.pred & 0x4); + carry = (data.u64 & 0x10000)>>16; break; - case U32_TYPE: - data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF) + - (src3_data.pred & 0x4); - carry = (data.u64 & 0x100000000) >> 32; + case U32_TYPE: + data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF) + (src3_data.pred & 0x4); + carry = (data.u64 & 0x100000000)>>32; break; - case U64_TYPE: + case U64_TYPE: data.s64 = src1_data.s64 + src2_data.s64 + (src3_data.pred & 0x4); break; - case F16_TYPE: - data.f16 = src1_data.f16 + src2_data.f16; - break; // assert(0); break; - case F32_TYPE: - data.f32 = src1_data.f32 + src2_data.f32; - break; - case F64_TYPE: - case FF64_TYPE: - data.f64 = src1_data.f64 + src2_data.f64; - break; - default: - assert(0); - break; - } - fesetround(orig_rm); + case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break; + case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; + case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; + default: assert(0); break; + } + fesetround( orig_rm ); - thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry ); } -void add_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; - int overflow = 0; - int carry = 0; +void add_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + int overflow = 0; + int carry = 0; - const operand_info &dst = - pI->dst(); // get operand info of sources and destination - const operand_info &src1 = - pI->src1(); // use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); //get operand info of sources and destination + const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - unsigned rounding_mode = pI->rounding_mode(); - int orig_rm = fegetround(); - switch (rounding_mode) { - case RN_OPTION: - break; - case RZ_OPTION: - fesetround(FE_TOWARDZERO); - break; - default: - assert(0); - break; - } + unsigned rounding_mode = pI->rounding_mode(); + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } - // performs addition. Sets carry and overflow if needed. - switch (i_type) { - case S8_TYPE: + //performs addition. Sets carry and overflow if needed. + switch ( i_type ) { + case S8_TYPE: data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF); - if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) == 0) { - overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; - } - carry = (data.u64 & 0x000000100) >> 8; + if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) == 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } + carry = (data.u64 & 0x000000100)>>8; break; - case S16_TYPE: + case S16_TYPE: data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF); - if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) == 0) { - overflow = - ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; - } - carry = (data.u64 & 0x000010000) >> 16; + if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) == 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } + carry = (data.u64 & 0x000010000)>>16; break; - case S32_TYPE: + case S32_TYPE: data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF); - if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) == 0) { - overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 - ? 0 - : 1; - } - carry = (data.u64 & 0x100000000) >> 32; + if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) == 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } + carry = (data.u64 & 0x100000000)>>32; break; - case S64_TYPE: + case S64_TYPE: data.s64 = src1_data.s64 + src2_data.s64; break; - case U8_TYPE: + case U8_TYPE: data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF); - carry = (data.u64 & 0x100) >> 8; + carry = (data.u64 & 0x100)>>8; break; - case U16_TYPE: + case U16_TYPE: data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF); - carry = (data.u64 & 0x10000) >> 16; + carry = (data.u64 & 0x10000)>>16; break; - case U32_TYPE: + case U32_TYPE: data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF); - carry = (data.u64 & 0x100000000) >> 32; + carry = (data.u64 & 0x100000000)>>32; break; - case U64_TYPE: + case U64_TYPE: data.u64 = src1_data.u64 + src2_data.u64; break; - case F16_TYPE: - data.f16 = src1_data.f16 + src2_data.f16; - break; // assert(0); break; - case F32_TYPE: - data.f32 = src1_data.f32 + src2_data.f32; - break; - case F64_TYPE: - case FF64_TYPE: - data.f64 = src1_data.f64 + src2_data.f64; - break; - default: - assert(0); - break; - } - fesetround(orig_rm); + case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break; + case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; + case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; + default: assert(0); break; + } + fesetround( orig_rm ); - thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry ); } -void addc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} +void addc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void and_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; +void and_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - // the way ptxplus handles predicates: 1 = false and 0 = true - if (i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) & ~(src2_data.pred)); - else - data.u64 = src1_data.u64 & src2_data.u64; - thread->set_operand_value(dst, data, i_type, thread, pI); + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) & ~(src2_data.pred)); + else + data.u64 = src1_data.u64 & src2_data.u64; + + thread->set_operand_value(dst,data, i_type, thread, pI); } -void andn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; +void andn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch (i_type) { - case B16_TYPE: - src2_data.u16 = ~src2_data.u16; - break; - case B32_TYPE: - src2_data.u32 = ~src2_data.u32; - break; - case B64_TYPE: - src2_data.u64 = ~src2_data.u64; - break; - default: + switch ( i_type ) { + case B16_TYPE: src2_data.u16 = ~src2_data.u16; break; + case B32_TYPE: src2_data.u32 = ~src2_data.u32; break; + case B64_TYPE: src2_data.u64 = ~src2_data.u64; break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - data.u64 = src1_data.u64 & src2_data.u64; + data.u64 = src1_data.u64 & src2_data.u64; - thread->set_operand_value(dst, data, i_type, thread, pI); + thread->set_operand_value(dst,data, i_type, thread, pI); } -void bar_callback(const inst_t *inst, ptx_thread_info *thread) { - unsigned ctaid = thread->get_cta_uid(); - unsigned barid = inst->bar_id; - unsigned value = thread->get_reduction_value(ctaid, barid); - const ptx_instruction *pI = dynamic_cast<const ptx_instruction *>(inst); - const operand_info &dst = pI->dst(); - ptx_reg_t data; - data.u32 = value; - thread->set_operand_value(dst, value, U32_TYPE, thread, pI); +void bar_callback( const inst_t* inst, ptx_thread_info* thread) +{ + unsigned ctaid = thread->get_cta_uid(); + unsigned barid = inst->bar_id; + unsigned value = thread->get_reduction_value(ctaid,barid); + const ptx_instruction *pI = dynamic_cast<const ptx_instruction*>(inst); + const operand_info &dst = pI->dst(); + ptx_reg_t data; + data.u32 = value; + thread->set_operand_value(dst,value, U32_TYPE, thread, pI); } -void atom_callback(const inst_t *inst, ptx_thread_info *thread) { - const ptx_instruction *pI = dynamic_cast<const ptx_instruction *>(inst); +void atom_callback( const inst_t* inst, ptx_thread_info* thread) +{ + const ptx_instruction *pI = dynamic_cast<const ptx_instruction*>(inst); - // "Decode" the output type - unsigned to_type = pI->get_type(); - size_t size; - int t; - type_info_key::type_decode(to_type, size, t); + // "Decode" the output type + unsigned to_type = pI->get_type(); + size_t size; + int t; + type_info_key::type_decode(to_type, size, t); - // Set up operand variables - ptx_reg_t data; // d - ptx_reg_t src1_data; // a - ptx_reg_t src2_data; // b - ptx_reg_t op_result; // temp variable to hold operation result + // Set up operand variables + ptx_reg_t data; // d + ptx_reg_t src1_data; // a + ptx_reg_t src2_data; // b + ptx_reg_t op_result; // temp variable to hold operation result - bool data_ready = false; + bool data_ready = false; - // Get operand info of sources and destination - const operand_info &dst = pI->dst(); // d - const operand_info &src1 = pI->src1(); // a - const operand_info &src2 = pI->src2(); // b + // Get operand info of sources and destination + const operand_info &dst = pI->dst(); // d + const operand_info &src1 = pI->src1(); // a + const operand_info &src2 = pI->src2(); // b - // Get operand values - src1_data = thread->get_operand_value(src1, src1, to_type, thread, 1); // a - if (dst.get_symbol()->type()) { - src2_data = thread->get_operand_value(src2, dst, to_type, thread, 1); // b - } else { - // This is the case whent he first argument (dest) is '_' - src2_data = thread->get_operand_value(src2, src1, to_type, thread, 1); // b - } + // Get operand values + src1_data = thread->get_operand_value(src1, src1, to_type, thread, 1); // a + if (dst.get_symbol()->type()){ + src2_data = thread->get_operand_value(src2, dst, to_type, thread, 1); // b + } else { + //This is the case whent he first argument (dest) is '_' + src2_data = thread->get_operand_value(src2, src1, to_type, thread, 1); // b + } - // Check state space - addr_t effective_address = src1_data.u64; - memory_space_t space = pI->get_space(); - if (space == undefined_space) { - // generic space - determine space via address - if (whichspace(effective_address) == global_space) { - effective_address = generic_to_global(effective_address); - space = global_space; - } else if (whichspace(effective_address) == shared_space) { - unsigned smid = thread->get_hw_sid(); - effective_address = generic_to_shared(smid, effective_address); - space = shared_space; - } else { - abort(); - } - } - assert(space == global_space || space == shared_space); + // Check state space + addr_t effective_address = src1_data.u64; + memory_space_t space = pI->get_space(); + if (space == undefined_space) { + // generic space - determine space via address + if( whichspace(effective_address) == global_space ) { + effective_address = generic_to_global(effective_address); + space = global_space; + } else if( whichspace(effective_address) == shared_space ) { + unsigned smid = thread->get_hw_sid(); + effective_address = generic_to_shared(smid,effective_address); + space = shared_space; + } else { + abort(); + } + } + assert( space == global_space || space == shared_space ); - memory_space *mem = NULL; - if (space == global_space) - mem = thread->get_global_memory(); - else if (space == shared_space) - mem = thread->m_shared_mem; - else - abort(); + memory_space *mem = NULL; + if(space == global_space) + mem = thread->get_global_memory(); + else if(space == shared_space) + mem = thread->m_shared_mem; + else + abort(); - // Copy value pointed to in operand 'a' into register 'd' - // (i.e. copy src1_data to dst) - mem->read(effective_address, size / 8, &data.s64); - if (dst.get_symbol()->type()) { - thread->set_operand_value(dst, data, to_type, thread, - pI); // Write value into register 'd' - } + // Copy value pointed to in operand 'a' into register 'd' + // (i.e. copy src1_data to dst) + mem->read(effective_address,size/8,&data.s64); + if (dst.get_symbol()->type()){ + thread->set_operand_value(dst, data, to_type, thread, pI); // Write value into register 'd' + } - // Get the atomic operation to be performed - unsigned m_atomic_spec = pI->get_atomic(); + // Get the atomic operation to be performed + unsigned m_atomic_spec = pI->get_atomic(); - switch (m_atomic_spec) { - // AND - case ATOMIC_AND: { - switch (to_type) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 & src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 & src2_data.s32; - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch (%x) with instruction\natom.AND " - "only accepts b32\n", - to_type); - assert(0); - break; + switch ( m_atomic_spec ) { + // AND + case ATOMIC_AND: + { + + switch ( to_type ) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 & src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 & src2_data.s32; + data_ready = true; + break; + default: + printf("Execution error: type mismatch (%x) with instruction\natom.AND only accepts b32\n", to_type); + assert(0); + break; + } + + break; } + // OR + case ATOMIC_OR: + { - break; - } - // OR - case ATOMIC_OR: { - switch (to_type) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 | src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 | src2_data.s32; - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch (%x) with instruction\natom.OR " - "only accepts b32\n", - to_type); - assert(0); - break; + switch ( to_type ) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 | src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 | src2_data.s32; + data_ready = true; + break; + default: + printf("Execution error: type mismatch (%x) with instruction\natom.OR only accepts b32\n", to_type); + assert(0); + break; + } + + break; } + // XOR + case ATOMIC_XOR: + { - break; - } - // XOR - case ATOMIC_XOR: { - switch (to_type) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 ^ src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 ^ src2_data.s32; - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch (%x) with instruction\natom.XOR " - "only accepts b32\n", - to_type); - assert(0); - break; + switch ( to_type ) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 ^ src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 ^ src2_data.s32; + data_ready = true; + break; + default: + printf("Execution error: type mismatch (%x) with instruction\natom.XOR only accepts b32\n", to_type); + assert(0); + break; + } + + break; } + // CAS + case ATOMIC_CAS: + { - break; - } - // CAS - case ATOMIC_CAS: { - ptx_reg_t src3_data; - const operand_info &src3 = pI->src3(); - src3_data = thread->get_operand_value(src3, dst, to_type, thread, 1); + ptx_reg_t src3_data; + const operand_info &src3 = pI->src3(); + src3_data = thread->get_operand_value(src3, dst, to_type, thread, 1); - switch (to_type) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = MY_CAS_I(data.u32, src2_data.u32, src3_data.u32); - data_ready = true; - break; - case B64_TYPE: - case U64_TYPE: - op_result.u64 = MY_CAS_I(data.u64, src2_data.u64, src3_data.u64); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_CAS_I(data.s32, src2_data.s32, src3_data.s32); - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch (%x) with instruction\natom.CAS " - "only accepts b32 and b64\n", - to_type); - assert(0); - break; + switch ( to_type ) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = MY_CAS_I(data.u32, src2_data.u32, src3_data.u32); + data_ready = true; + break; + case B64_TYPE: + case U64_TYPE: + op_result.u64 = MY_CAS_I(data.u64, src2_data.u64, src3_data.u64); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_CAS_I(data.s32, src2_data.s32, src3_data.s32); + data_ready = true; + break; + default: + printf("Execution error: type mismatch (%x) with instruction\natom.CAS only accepts b32 and b64\n", to_type); + assert(0); + break; + } + + break; } + // EXCH + case ATOMIC_EXCH: + { + switch ( to_type ) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = MY_EXCH(data.u32, src2_data.u32); + data_ready = true; + break; + case B64_TYPE: + case U64_TYPE: + op_result.u64 = MY_EXCH(data.u64, src2_data.u64); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_EXCH(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf("Execution error: type mismatch (%x) with instruction\natom.EXCH only accepts b32\n", to_type); + assert(0); + break; + } - break; - } - // EXCH - case ATOMIC_EXCH: { - switch (to_type) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = MY_EXCH(data.u32, src2_data.u32); - data_ready = true; - break; - case B64_TYPE: - case U64_TYPE: - op_result.u64 = MY_EXCH(data.u64, src2_data.u64); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_EXCH(data.s32, src2_data.s32); - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch (%x) with instruction\natom.EXCH " - "only accepts b32\n", - to_type); - assert(0); - break; + break; } + // ADD + case ATOMIC_ADD: + { - break; - } - // ADD - case ATOMIC_ADD: { - switch (to_type) { - case U32_TYPE: - op_result.u32 = data.u32 + src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 + src2_data.s32; - data_ready = true; - break; - case U64_TYPE: - op_result.u64 = data.u64 + src2_data.u64; - data_ready = true; - break; - case F32_TYPE: - op_result.f32 = data.f32 + src2_data.f32; - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch with instruction\natom.ADD only " - "accepts u32, s32, u64, and f32\n"); - assert(0); - break; + switch ( to_type ) { + case U32_TYPE: + op_result.u32 = data.u32 + src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 + src2_data.s32; + data_ready = true; + break; + case U64_TYPE: + op_result.u64 = data.u64 + src2_data.u64; + data_ready = true; + break; + case F32_TYPE: + op_result.f32 = data.f32 + src2_data.f32; + data_ready = true; + break; + default: + printf("Execution error: type mismatch with instruction\natom.ADD only accepts u32, s32, u64, and f32\n"); + assert(0); + break; + } + + break; } + // INC + case ATOMIC_INC: + { + switch ( to_type ) { + case U32_TYPE: + op_result.u32 = MY_INC_I(data.u32, src2_data.u32); + data_ready = true; + break; + default: + printf("Execution error: type mismatch with instruction\natom.INC only accepts u32 and s32\n"); + assert(0); + break; + } - break; - } - // INC - case ATOMIC_INC: { - switch (to_type) { - case U32_TYPE: - op_result.u32 = MY_INC_I(data.u32, src2_data.u32); - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch with instruction\natom.INC only " - "accepts u32 and s32\n"); - assert(0); - break; + break; } + // DEC + case ATOMIC_DEC: + { + switch ( to_type ) { + case U32_TYPE: + op_result.u32 = MY_DEC_I(data.u32, src2_data.u32); + data_ready = true; + break; + default: + printf("Execution error: type mismatch with instruction\natom.DEC only accepts u32 and s32\n"); + assert(0); + break; + } - break; - } - // DEC - case ATOMIC_DEC: { - switch (to_type) { - case U32_TYPE: - op_result.u32 = MY_DEC_I(data.u32, src2_data.u32); - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch with instruction\natom.DEC only " - "accepts u32 and s32\n"); - assert(0); - break; + break; } + // MIN + case ATOMIC_MIN: + { + switch ( to_type ) { + case U32_TYPE: + op_result.u32 = MY_MIN_I(data.u32, src2_data.u32); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_MIN_I(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf("Execution error: type mismatch with instruction\natom.MIN only accepts u32 and s32\n"); + assert(0); + break; + } - break; - } - // MIN - case ATOMIC_MIN: { - switch (to_type) { - case U32_TYPE: - op_result.u32 = MY_MIN_I(data.u32, src2_data.u32); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_MIN_I(data.s32, src2_data.s32); - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch with instruction\natom.MIN only " - "accepts u32 and s32\n"); - assert(0); - break; + break; } + // MAX + case ATOMIC_MAX: + { + switch ( to_type ) { + case U32_TYPE: + op_result.u32 = MY_MAX_I(data.u32, src2_data.u32); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_MAX_I(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf("Execution error: type mismatch with instruction\natom.MAX only accepts u32 and s32\n"); + assert(0); + break; + } - break; - } - // MAX - case ATOMIC_MAX: { - switch (to_type) { - case U32_TYPE: - op_result.u32 = MY_MAX_I(data.u32, src2_data.u32); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_MAX_I(data.s32, src2_data.s32); - data_ready = true; - break; - default: - printf( - "Execution error: type mismatch with instruction\natom.MAX only " - "accepts u32 and s32\n"); - assert(0); - break; + break; } + // DEFAULT + default: + { + assert(0); + break; + } + } - break; - } - // DEFAULT - default: { + // Write operation result into memory + // (i.e. copy src1_data to dst) + if ( data_ready ) { + mem->write(effective_address,size/8,&op_result.s64,thread,pI); + } else { + printf("Execution error: data_ready not set\n"); assert(0); - break; - } - } - - // Write operation result into memory - // (i.e. copy src1_data to dst) - if (data_ready) { - mem->write(effective_address, size / 8, &op_result.s64, thread, pI); - } else { - printf("Execution error: data_ready not set\n"); - assert(0); - } + } } -// atom_impl will now result in a callback being called in mem_ctrl_pop -// (gpu-sim.c) -void atom_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - // SYNTAX - // atom.space.operation.type d, a, b[, c]; (now read in callback) +// atom_impl will now result in a callback being called in mem_ctrl_pop (gpu-sim.c) +void atom_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + // SYNTAX + // atom.space.operation.type d, a, b[, c]; (now read in callback) - // obtain memory space of the operation - memory_space_t space = pI->get_space(); + // obtain memory space of the operation + memory_space_t space = pI->get_space(); - // get the memory address - const operand_info &src1 = pI->src1(); - // const operand_info &dst = pI->dst(); // not needed for effective address - // calculation - unsigned i_type = pI->get_type(); - ptx_reg_t src1_data; - src1_data = thread->get_operand_value(src1, src1, i_type, thread, 1); - addr_t effective_address = src1_data.u64; + // get the memory address + const operand_info &src1 = pI->src1(); + // const operand_info &dst = pI->dst(); // not needed for effective address calculation + unsigned i_type = pI->get_type(); + ptx_reg_t src1_data; + src1_data = thread->get_operand_value(src1, src1, i_type, thread, 1); + addr_t effective_address = src1_data.u64; - addr_t effective_address_final; + addr_t effective_address_final; - // handle generic memory space by converting it to global - if (space == undefined_space) { - if (whichspace(effective_address) == global_space) { - effective_address_final = generic_to_global(effective_address); - space = global_space; - } else if (whichspace(effective_address) == shared_space) { - unsigned smid = thread->get_hw_sid(); - effective_address_final = generic_to_shared(smid, effective_address); - space = shared_space; - } else { - abort(); - } - } else { - assert(space == global_space || space == shared_space); - effective_address_final = effective_address; - } + // handle generic memory space by converting it to global + if ( space == undefined_space ) { + if( whichspace(effective_address) == global_space ) { + effective_address_final = generic_to_global(effective_address); + space = global_space; + } else if( whichspace(effective_address) == shared_space ) { + unsigned smid = thread->get_hw_sid(); + effective_address_final = generic_to_shared(smid,effective_address); + space = shared_space; + } else { + abort(); + } + } else { + assert( space == global_space || space == shared_space ); + effective_address_final = effective_address; + } - // Check state space - assert(space == global_space || space == shared_space); + // Check state space + assert( space == global_space || space == shared_space ); - thread->m_last_effective_address = effective_address_final; - thread->m_last_memory_space = space; - thread->m_last_dram_callback.function = atom_callback; - thread->m_last_dram_callback.instruction = pI; + thread->m_last_effective_address = effective_address_final; + thread->m_last_memory_space = space; + thread->m_last_dram_callback.function = atom_callback; + thread->m_last_dram_callback.instruction = pI; } -void bar_impl(const ptx_instruction *pIin, ptx_thread_info *thread) { - ptx_instruction *pI = const_cast<ptx_instruction *>(pIin); - unsigned bar_op = pI->barrier_op(); - unsigned red_op = pI->get_atomic(); - unsigned ctaid = thread->get_cta_uid(); +void bar_impl( const ptx_instruction *pIin, ptx_thread_info *thread ) +{ + ptx_instruction * pI = const_cast<ptx_instruction *>(pIin); + unsigned bar_op = pI->barrier_op(); + unsigned red_op = pI->get_atomic(); + unsigned ctaid = thread->get_cta_uid(); - switch (bar_op) { - case SYNC_OPTION: { - if (pI->get_num_operands() > 1) { - const operand_info &op0 = pI->dst(); - const operand_info &op1 = pI->src1(); - ptx_reg_t op0_data; - ptx_reg_t op1_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); - pI->set_bar_count(op1_data.u32); - } else { - const operand_info &op0 = pI->dst(); - ptx_reg_t op0_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); - } - break; - } - case ARRIVE_OPTION: { - const operand_info &op0 = pI->dst(); - const operand_info &op1 = pI->src1(); - ptx_reg_t op0_data; - ptx_reg_t op1_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); - pI->set_bar_count(op1_data.u32); - break; - } - case RED_OPTION: { - if (pI->get_num_operands() > 3) { - const operand_info &op1 = pI->src1(); - const operand_info &op2 = pI->src2(); - const operand_info &op3 = pI->src3(); - ptx_reg_t op1_data; - ptx_reg_t op2_data; - ptx_reg_t op3_data; - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - op2_data = thread->get_operand_value(op2, op2, U32_TYPE, thread, 1); - op3_data = thread->get_operand_value(op3, op3, PRED_TYPE, thread, 1); - op3_data.u32 = !(op3_data.pred & 0x0001); - pI->set_bar_id(op1_data.u32); - pI->set_bar_count(op2_data.u32); - switch (red_op) { - case ATOMIC_POPC: - thread->popc_reduction(ctaid, op1_data.u32, op3_data.u32); - break; - case ATOMIC_AND: - thread->and_reduction(ctaid, op1_data.u32, op3_data.u32); - break; - case ATOMIC_OR: - thread->or_reduction(ctaid, op1_data.u32, op3_data.u32); - break; - default: - abort(); - break; - } - } else { - const operand_info &op1 = pI->src1(); - const operand_info &op2 = pI->src2(); - ptx_reg_t op1_data; - ptx_reg_t op2_data; - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - op2_data = thread->get_operand_value(op2, op2, PRED_TYPE, thread, 1); - op2_data.u32 = !(op2_data.pred & 0x0001); - pI->set_bar_id(op1_data.u32); - pI->set_bar_count(thread->get_ntid().x * thread->get_ntid().y * - thread->get_ntid().z); - switch (red_op) { - case ATOMIC_POPC: - thread->popc_reduction(ctaid, op1_data.u32, op2_data.u32); - break; - case ATOMIC_AND: - thread->and_reduction(ctaid, op1_data.u32, op2_data.u32); - break; - case ATOMIC_OR: - thread->or_reduction(ctaid, op1_data.u32, op2_data.u32); - break; - default: - abort(); - break; - } - } - break; - } - default: - abort(); - break; - } + switch(bar_op){ + case SYNC_OPTION: + { + if(pI->get_num_operands()>1){ + const operand_info &op0 = pI->dst(); + const operand_info &op1 = pI->src1(); + ptx_reg_t op0_data; + ptx_reg_t op1_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + pI->set_bar_count(op1_data.u32); + }else{ + const operand_info &op0 = pI->dst(); + ptx_reg_t op0_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + } + break; + } + case ARRIVE_OPTION: + { + const operand_info &op0 = pI->dst(); + const operand_info &op1 = pI->src1(); + ptx_reg_t op0_data; + ptx_reg_t op1_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + pI->set_bar_count(op1_data.u32); + break; + } + case RED_OPTION: + { + if(pI->get_num_operands()>3){ + const operand_info &op1 = pI->src1(); + const operand_info &op2 = pI->src2(); + const operand_info &op3 = pI->src3(); + ptx_reg_t op1_data; + ptx_reg_t op2_data; + ptx_reg_t op3_data; + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + op2_data = thread->get_operand_value(op2, op2, U32_TYPE, thread, 1); + op3_data = thread->get_operand_value(op3, op3, PRED_TYPE, thread, 1); + op3_data.u32=!(op3_data.pred & 0x0001); + pI->set_bar_id(op1_data.u32); + pI->set_bar_count(op2_data.u32); + switch(red_op){ + case ATOMIC_POPC: + thread->popc_reduction(ctaid,op1_data.u32,op3_data.u32); + break; + case ATOMIC_AND: + thread->and_reduction(ctaid,op1_data.u32,op3_data.u32); + break; + case ATOMIC_OR: + thread->or_reduction(ctaid,op1_data.u32,op3_data.u32); + break; + default: + abort(); + break; + } + }else{ + const operand_info &op1 = pI->src1(); + const operand_info &op2 = pI->src2(); + ptx_reg_t op1_data; + ptx_reg_t op2_data; + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + op2_data = thread->get_operand_value(op2, op2, PRED_TYPE, thread, 1); + op2_data.u32=!(op2_data.pred & 0x0001); + pI->set_bar_id(op1_data.u32); + pI->set_bar_count(thread->get_ntid().x * thread->get_ntid().y * thread->get_ntid().z); + switch(red_op){ + case ATOMIC_POPC: + thread->popc_reduction(ctaid,op1_data.u32,op2_data.u32); + break; + case ATOMIC_AND: + thread->and_reduction(ctaid,op1_data.u32,op2_data.u32); + break; + case ATOMIC_OR: + thread->or_reduction(ctaid,op1_data.u32,op2_data.u32); + break; + default: + abort(); + break; + } + } + break; + } + default: + abort(); + break; + } - thread->m_last_dram_callback.function = bar_callback; - thread->m_last_dram_callback.instruction = pIin; + thread->m_last_dram_callback.function = bar_callback; + thread->m_last_dram_callback.instruction = pIin; } -void bfe_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - unsigned i_type = pI->get_type(); - unsigned msb = (i_type == U32_TYPE || i_type == S32_TYPE) ? 31 : 63; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t src = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - ptx_reg_t data; - unsigned pos = b.u32 & 0xFF; - unsigned len = c.u32 & 0xFF; - switch (i_type) { - case U32_TYPE: { - unsigned mask; - data.u32 = src.u32 >> pos; - mask = 0xFFFFFFFF >> (32 - len); - data.u32 &= mask; - break; - } - case U64_TYPE: { - unsigned long mask; - data.u64 = src.u64 >> pos; - mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - data.u64 &= mask; - break; - } - case S32_TYPE: { - unsigned mask; - unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (src.s32 >> min) & 0x1; - data.s32 = src.s32 >> pos; - if (sbit > 0) { - mask = 0xFFFFFFFF << len; - data.s32 |= mask; - } else { - mask = 0xFFFFFFFF >> (32 - len); - data.s32 &= mask; - } - break; - } - case S64_TYPE: { - unsigned long mask; - unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (src.s64 >> min) & 0x1; - data.s64 = src.s64 >> pos; - if (sbit > 0) { - mask = 0xFFFFFFFFFFFFFFFF << len; - data.s64 |= mask; - } else { - mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - data.s64 &= mask; - } - break; - } - default: - printf("Operand type not supported for BFE instruction.\n"); - abort(); - return; - } - thread->set_operand_value(dst, data, i_type, thread, pI); +void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + unsigned i_type = pI->get_type(); + unsigned msb = (i_type == U32_TYPE || i_type == S32_TYPE) ? 31 : 63; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t src = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + ptx_reg_t data; + unsigned pos = b.u32 & 0xFF; + unsigned len = c.u32 & 0xFF; + switch (i_type) + { + case U32_TYPE: + { + unsigned mask; + data.u32 = src.u32 >> pos; + mask = 0xFFFFFFFF >> (32 - len); + data.u32 &= mask; + break; + } + case U64_TYPE: + { + unsigned long mask; + data.u64 = src.u64 >> pos; + mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); + data.u64 &= mask; + break; + } + case S32_TYPE: + { + unsigned mask; + unsigned min = MY_MIN_I(pos + len - 1, msb); + unsigned sbit = len == 0 ? 0 : (src.s32 >> min) & 0x1; + data.s32 = src.s32 >> pos; + if (sbit > 0) + { + mask = 0xFFFFFFFF << len; + data.s32 |= mask; + } + else + { + mask = 0xFFFFFFFF >> (32 - len); + data.s32 &= mask; + } + break; + } + case S64_TYPE: + { + unsigned long mask; + unsigned min = MY_MIN_I(pos + len - 1, msb); + unsigned sbit = len == 0 ? 0 : (src.s64 >> min) & 0x1; + data.s64 = src.s64 >> pos; + if (sbit > 0) + { + mask = 0xFFFFFFFFFFFFFFFF << len; + data.s64 |= mask; + } + else + { + mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); + data.s64 &= mask; + } + break; + } + default: + printf("Operand type not supported for BFE instruction.\n"); + abort(); + return; + } + thread->set_operand_value(dst, data, i_type, thread, pI); } -void bfi_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - int i, max; - ptx_reg_t src1_data, src2_data; - ptx_reg_t src3_data, src4_data, data; +void bfi_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { + int i,max; + ptx_reg_t src1_data, src2_data; + ptx_reg_t src3_data, src4_data, data; - const operand_info &dst = - pI->dst(); // get operand info of sources and destination - const operand_info &src1 = - pI->src1(); // use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - const operand_info &src4 = pI->src4(); + const operand_info &dst = pI->dst(); //get operand info of sources and destination + const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + const operand_info &src4 = pI->src4(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - src4_data = thread->get_operand_value(src4, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + src4_data = thread->get_operand_value(src4, dst, i_type, thread, 1); - switch (i_type) { - case B32_TYPE: + switch ( i_type ) { + case B32_TYPE: max = 32; break; - case B64_TYPE: + case B64_TYPE: max = 64; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } - data = src2_data; - unsigned pos = src3_data.u32 & 0xFF; - unsigned len = src4_data.u32 & 0xFF; - for (i = 0; i < len && pos + i < max; i++) { - data.u32 = (~((0x00000001) << (pos + i))) & data.u32; - data.u32 = data.u32 | ((src1_data.u32 & ((0x00000001) << (i))) << (pos)); - } - thread->set_operand_value(dst, data, i_type, thread, pI); -} -void bfind_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); + } + data=src2_data; + unsigned pos = src3_data.u32 & 0xFF; + unsigned len = src4_data.u32 & 0xFF; + for(i=0;i<len && pos+i<max;i++){ + data.u32=(~((0x00000001)<<(pos+i)))&data.u32; + data.u32=data.u32|((src1_data.u32&((0x00000001)<<(i)))<<(pos)); + } + thread->set_operand_value(dst, data, i_type, thread, pI); } +void bfind_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void bra_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = - thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void bra_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); - thread->m_branch_taken = true; - thread->set_npc(target_pc); + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void brx_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = - thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void brx_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); - thread->m_branch_taken = true; - thread->set_npc(target_pc); + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void break_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &target = thread->pop_breakaddr(); - ptx_reg_t target_pc = - thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void break_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &target = thread->pop_breakaddr(); + ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); - thread->m_branch_taken = true; - thread->set_npc(target_pc); + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void breakaddr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &target = pI->dst(); - thread->push_breakaddr(target); - assert( - pI->has_pred() == - false); // pdom analysis cannot handle if this instruction is predicated +void breakaddr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &target = pI->dst(); + thread->push_breakaddr(target); + assert(pI->has_pred() == false); // pdom analysis cannot handle if this instruction is predicated } -void brev_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); +void brev_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - unsigned msb; - switch (i_type) { - case B32_TYPE: - msb = 31; - for (unsigned i = 0; i <= msb; i++) { - if ((src1_data.u32 & (1 << i))) data.u32 |= 1 << (msb - i); - } - break; - case B64_TYPE: - msb = 63; - for (unsigned i = 0; i <= msb; i++) { - if ((src1_data.u64 & (1 << i))) data.u64 |= 1 << (msb - i); - } - break; - default: - assert(0); - } - thread->set_operand_value(dst, data, i_type, thread, pI); -} -void brkpt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); + unsigned msb; + switch(i_type){ + case B32_TYPE: + msb = 31; + for (unsigned i=0; i<=msb; i++) { + if((src1_data.u32 & (1 << i))) + data.u32 |= 1 << (msb - i); + } + break; + case B64_TYPE: + msb = 63; + for (unsigned i=0; i<=msb; i++) { + if((src1_data.u64 & (1 << i))) + data.u64 |= 1 << (msb - i); + } + break; + default: assert(0); + } + thread->set_operand_value(dst,data, i_type, thread, pI); } +void brkpt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } unsigned trunc(unsigned num, unsigned precision) { - int mask = 1, latest_one = -1; - unsigned data = num; - for (unsigned j = 0; j < sizeof(unsigned) * 8; j++) { - int bit = data & mask; - if (bit == 1) latest_one = j; - data >>= 1; - } - if (latest_one >= precision) { - // round_up is 1 if the most significant truncated digit is a 1, otherwise - // it is 0 - // int round_up = (num & (1 << (latest_one-precision))) >> - // (latest_one-precision); - // unsigned shifted_output = num >> (latest_one-precision+1); - // if shifted_output is a number like 1111, don't round up - // if (shifted_output == (pow(2,precision)-1)) round_up = 0; - // num = shifted_output + round_up; - num >>= (latest_one - precision + 1); - } - return num; + int mask = 1, latest_one = -1; + unsigned data = num; + for (unsigned j = 0; j < sizeof(unsigned)*8; j++) { + int bit = data & mask; + if (bit == 1) latest_one = j; + data >>= 1; + } + if (latest_one >= precision) { + // round_up is 1 if the most significant truncated digit is a 1, otherwise it is 0 + //int round_up = (num & (1 << (latest_one-precision))) >> (latest_one-precision); + //unsigned shifted_output = num >> (latest_one-precision+1); + // if shifted_output is a number like 1111, don't round up + //if (shifted_output == (pow(2,precision)-1)) round_up = 0; + //num = shifted_output + round_up; + num >>= (latest_one-precision+1); + } + return num; } -void mapping(int thread, int wmma_type, int wmma_layout, int type, int index, - int stride, int &row, int &col, int &assg_offset) { - int offset; - int c_row_offset[] = {0, 8, 0, 8, 4, 12, 4, 12}; - int c_col_offset[] = {0, 0, 8, 8, 0, 0, 8, 8}; - int c_tg_inside_row_offset[] = {0, 1, 0, 1}; - int c_tg_inside_col_offset[] = {0, 0, 2, 2}; - int c_inside_row_offset[] = {0, 0, 2, 2, 0, 0, 2, 2}; - int c_inside_col_offset[] = {0, 1, 0, 1, 4, 5, 4, 5}; +void mapping(int thread,int wmma_type,int wmma_layout,int type,int index,int stride,int &row,int &col,int &assg_offset){ + int offset; + int c_row_offset[]={0,8,0,8,4,12,4,12}; + int c_col_offset[]={0,0,8,8,0,0,8,8}; + int c_tg_inside_row_offset[]={0,1,0,1}; + int c_tg_inside_col_offset[]={0,0,2,2}; + int c_inside_row_offset[]={0,0,2,2,0,0,2,2}; + int c_inside_col_offset[]={0,1,0,1,4,5,4,5}; - offset = thread_group_offset(thread, wmma_type, wmma_layout, type, stride); + offset=thread_group_offset(thread,wmma_type,wmma_layout,type,stride); - if (wmma_type == LOAD_A) { - if (wmma_layout == ROW) { - offset += index + 8 * ((thread % 16) / 8); - } else { - offset += 64 * (index / 4) + index % 4 + 128 * ((thread % 16) / 8); - } - offset = (offset / 16) * stride + offset % 16; - assg_offset = index + 8 * ((thread % 16) / 8); - } else if (wmma_type == LOAD_B) { - if (wmma_layout == ROW) { - offset += 64 * (index / 4) + index % 4 + 128 * ((thread % 16) / 8); - } else { - offset += index + 8 * ((thread % 16) / 8); - } - offset = (offset / 16) * stride + offset % 16; - assg_offset = index + 8 * ((thread % 16) / 8); - } else if (wmma_type == LOAD_C) { - if (type == F16_TYPE) { - row = c_row_offset[thread / 4] + thread % 4; - col = c_col_offset[thread / 4] + index; - } else { - row = c_row_offset[thread / 4] + c_tg_inside_row_offset[thread % 4] + - c_inside_row_offset[index]; - col = c_col_offset[thread / 4] + c_tg_inside_col_offset[thread % 4] + - c_inside_col_offset[index]; - } - assg_offset = index; - } + if(wmma_type==LOAD_A){ + if(wmma_layout==ROW){ + offset+=index+8*((thread%16)/8); + } + else{ + offset+=64*(index/4)+index%4+128*((thread%16)/8); + } + offset=(offset/16)*stride+offset%16; + assg_offset=index+8*((thread%16)/8); + } + else if(wmma_type==LOAD_B){ + if(wmma_layout==ROW){ + offset+=64*(index/4)+index%4+128*((thread%16)/8); + } + else{ + offset+=index+8*((thread%16)/8); + } + offset=(offset/16)*stride+offset%16; + assg_offset=index+8*((thread%16)/8); + } + else if( wmma_type==LOAD_C){ + if(type==F16_TYPE){ + row=c_row_offset[thread/4]+thread%4; + col=c_col_offset[thread/4]+index; + } + else{ + row=c_row_offset[thread/4]+c_tg_inside_row_offset[thread%4]+c_inside_row_offset[index]; + col=c_col_offset[thread/4]+c_tg_inside_col_offset[thread%4]+c_inside_col_offset[index]; + } + assg_offset=index; + } - if (wmma_type == LOAD_A || wmma_type == LOAD_B) { - if (wmma_layout == ROW) { - row = offset / 16; - col = offset % 16; - } else { - col = offset / 16; - row = offset % 16; - } - } + if(wmma_type==LOAD_A||wmma_type==LOAD_B){ + if(wmma_layout==ROW){ + row=offset/16; + col=offset%16; + } + else{ + col=offset/16; + row=offset%16; + } + } } -void mma_impl(const ptx_instruction *pI, core_t *core, warp_inst_t inst) { - int i, j, k, thrd; - int row, col, offset; - ptx_reg_t matrix_a[16][16]; - ptx_reg_t matrix_b[16][16]; - ptx_reg_t matrix_c[16][16]; - ptx_reg_t matrix_d[16][16]; - ptx_reg_t src_data; - ptx_thread_info *thread; - - unsigned a_layout = pI->get_wmma_layout(0); - unsigned b_layout = pI->get_wmma_layout(1); - unsigned type = pI->get_type(); - unsigned type2 = pI->get_type2(); - int tid; - const operand_info &dst = pI->operand_lookup(0); - - if (core->get_gpu()->is_functional_sim()) - tid = inst.warp_id_func() * core->get_warp_size(); - else - tid = inst.warp_id() * core->get_warp_size(); - float temp; - half temp2; - - for (thrd = 0; thrd < core->get_warp_size(); thrd++) { - thread = core->get_thread_info()[tid + thrd]; - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("THREAD=%d\n:", thrd); - for (int operand_num = 1; operand_num <= 3; operand_num++) { - const operand_info &src_a = pI->operand_lookup(operand_num); - unsigned nelem = src_a.get_vect_nelem(); - ptx_reg_t v[8]; - thread->get_vector_operand_values(src_a, v, nelem); - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - printf("Thread%d_Iteration=%d\n:", thrd, operand_num); - for (k = 0; k < nelem; k++) { - printf("%llx ", v[k].u64); - } - printf("\n"); - } - ptx_reg_t nw_v[16]; - int hex_val; - - if (!((operand_num == 3) && (type2 == F32_TYPE))) { - for (k = 0; k < 2 * nelem; k++) { - if (k % 2 == 1) - hex_val = (v[k / 2].s64 & 0xffff); - else - hex_val = ((v[k / 2].s64 & 0xffff0000) >> 16); - nw_v[k].f16 = *((half *)&hex_val); - } - } - if (!((operand_num == 3) && (type2 == F32_TYPE))) { - for (k = 0; k < 2 * nelem; k++) { - temp = nw_v[k].f16; - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("%.2f ", temp); - } - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) printf("\n"); - } else { - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - for (k = 0; k < 8; k++) { - printf("%.2f ", v[k].f32); - } - printf("\n"); - } - } - switch (operand_num) { - case 1: // operand 1 - for (k = 0; k < 8; k++) { - mapping(thrd, LOAD_A, a_layout, F16_TYPE, k, 16, row, col, offset); - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("A:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, - offset); - matrix_a[row][col] = nw_v[offset]; - } - break; - case 2: // operand 2 - for (k = 0; k < 8; k++) { - mapping(thrd, LOAD_B, b_layout, F16_TYPE, k, 16, row, col, offset); - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("B:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, - offset); - matrix_b[row][col] = nw_v[offset]; - } - break; - case 3: // operand 3 - for (k = 0; k < 8; k++) { - mapping(thrd, LOAD_C, ROW, type2, k, 16, row, col, offset); - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("C:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, - offset); - if (type2 != F16_TYPE) { - matrix_c[row][col] = v[offset]; - } else { - matrix_c[row][col] = nw_v[offset]; - } - } - break; - default: - printf("Invalid Operand Index\n"); - } - } - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) printf("\n"); - } - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - printf("MATRIX_A\n"); - for (i = 0; i < 16; i++) { - for (j = 0; j < 16; j++) { - temp = matrix_a[i][j].f16; - printf("%.2f ", temp); - } - printf("\n"); - } - printf("MATRIX_B\n"); - for (i = 0; i < 16; i++) { - for (j = 0; j < 16; j++) { - temp = matrix_b[i][j].f16; - printf("%.2f ", temp); - } - printf("\n"); - } - printf("MATRIX_C\n"); - for (i = 0; i < 16; i++) { - for (j = 0; j < 16; j++) { - if (type2 == F16_TYPE) { - temp = matrix_c[i][j].f16; - printf("%.2f ", temp); - } else - printf("%.2f ", matrix_c[i][j].f32); - } - printf("\n"); - } - } - for (i = 0; i < 16; i++) { - for (j = 0; j < 16; j++) { - matrix_d[i][j].f16 = 0; - } - } - - for (i = 0; i < 16; i++) { - for (j = 0; j < 16; j++) { - for (k = 0; k < 16; k++) { - matrix_d[i][j].f16 = - matrix_d[i][j].f16 + matrix_a[i][k].f16 * matrix_b[k][j].f16; - } - if ((type == F16_TYPE) && (type2 == F16_TYPE)) - matrix_d[i][j].f16 += matrix_c[i][j].f16; - else if ((type == F32_TYPE) && (type2 == F16_TYPE)) { - temp2 = matrix_d[i][j].f16 + matrix_c[i][j].f16; - temp = temp2; - matrix_d[i][j].f32 = temp; - } else if ((type == F16_TYPE) && (type2 == F32_TYPE)) { - temp = matrix_d[i][j].f16; - temp += matrix_c[i][j].f32; - matrix_d[i][j].f16 = half(temp); - } else { - temp = matrix_d[i][j].f16; - temp += matrix_c[i][j].f32; - matrix_d[i][j].f32 = temp; - } - } - } - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - printf("MATRIX_D\n"); - for (i = 0; i < 16; i++) { - for (j = 0; j < 16; j++) { - if (type == F16_TYPE) { - temp = matrix_d[i][j].f16; - printf("%.2f ", temp); - } else - printf("%.2f ", matrix_d[i][j].f32); - } - printf("\n"); - } - } - for (thrd = 0; thrd < core->get_warp_size(); thrd++) { - int row_t[8]; - int col_t[8]; - for (k = 0; k < 8; k++) { - mapping(thrd, LOAD_C, ROW, type, k, 16, row_t[k], col_t[k], offset); - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma:store:row:%d,col%d\n", row_t[k], col_t[k]); - } - thread = core->get_thread_info()[tid + thrd]; - - if (type == F32_TYPE) { - thread->set_wmma_vector_operand_values( - dst, matrix_d[row_t[0]][col_t[0]], matrix_d[row_t[1]][col_t[1]], - matrix_d[row_t[2]][col_t[2]], matrix_d[row_t[3]][col_t[3]], - matrix_d[row_t[4]][col_t[4]], matrix_d[row_t[5]][col_t[5]], - matrix_d[row_t[6]][col_t[6]], matrix_d[row_t[7]][col_t[7]]); +void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) +{ + int i,j,k,thrd; + int row,col,offset; + ptx_reg_t matrix_a[16][16]; + ptx_reg_t matrix_b[16][16]; + ptx_reg_t matrix_c[16][16]; + ptx_reg_t matrix_d[16][16]; + ptx_reg_t src_data; + ptx_thread_info *thread; - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - printf("thread%d:", thrd); - for (k = 0; k < 8; k++) { - printf("%.2f ", matrix_d[row_t[k]][col_t[k]].f32); - } - printf("\n"); - } - } else if (type == F16_TYPE) { - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - printf("thread%d:", thrd); - for (k = 0; k < 8; k++) { - temp = matrix_d[row_t[k]][col_t[k]].f16; - printf("%.2f ", temp); - } - printf("\n"); + unsigned a_layout = pI->get_wmma_layout(0); + unsigned b_layout = pI->get_wmma_layout(1); + unsigned type = pI->get_type(); + unsigned type2 = pI->get_type2(); + int tid ; + const operand_info &dst = pI->operand_lookup(0); + + if(core->get_gpu()->is_functional_sim()) + tid= inst.warp_id_func()*core->get_warp_size(); + else + tid= inst.warp_id()*core->get_warp_size(); + float temp; + half temp2; + + for (thrd=0; thrd < core->get_warp_size(); thrd++){ + thread = core->get_thread_info()[tid+thrd]; + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("THREAD=%d\n:",thrd); + for(int operand_num=1;operand_num<=3;operand_num++){ + const operand_info &src_a= pI->operand_lookup(operand_num); + unsigned nelem = src_a.get_vect_nelem(); + ptx_reg_t v[8]; + thread->get_vector_operand_values( src_a, v, nelem ); + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ + printf("Thread%d_Iteration=%d\n:", thrd, operand_num); + for(k = 0; k < nelem; k++){ + printf("%llx ",v[k].u64); + } + printf("\n"); + } + ptx_reg_t nw_v[16]; + int hex_val; - printf("thread%d:", thrd); - for (k = 0; k < 8; k++) { - printf("%x ", (unsigned int)matrix_d[row_t[k]][col_t[k]].f16); - } - printf("\n"); - } - ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4; - nw_data1.s64 = ((matrix_d[row_t[0]][col_t[0]].s64 & 0xffff)) | - ((matrix_d[row_t[1]][col_t[1]].s64 & 0xffff) << 16); - nw_data2.s64 = ((matrix_d[row_t[2]][col_t[2]].s64 & 0xffff)) | - ((matrix_d[row_t[3]][col_t[3]].s64 & 0xffff) << 16); - nw_data3.s64 = ((matrix_d[row_t[4]][col_t[4]].s64 & 0xffff)) | - ((matrix_d[row_t[5]][col_t[5]].s64 & 0xffff) << 16); - nw_data4.s64 = ((matrix_d[row_t[6]][col_t[6]].s64 & 0xffff)) | - ((matrix_d[row_t[7]][col_t[7]].s64 & 0xffff) << 16); - thread->set_vector_operand_values(dst, nw_data1, nw_data2, nw_data3, - nw_data4); - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("thread%d=%llx,%llx,%llx,%llx", thrd, nw_data1.s64, nw_data2.s64, - nw_data3.s64, nw_data4.s64); + if(!((operand_num==3)&&(type2==F32_TYPE))){ + for(k=0;k<2*nelem;k++){ + if(k%2==1) + hex_val=(v[k/2].s64&0xffff); + else + hex_val=((v[k/2].s64&0xffff0000)>>16); + nw_v[k].f16 =*((half *)&hex_val); + } + } + if(!((operand_num==3)&&(type2==F32_TYPE))){ + for(k=0;k<2*nelem;k++){ + temp=nw_v[k].f16; + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("%.2f ",temp); + } + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("\n"); + } + else{ + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ + for(k=0;k<8;k++){ + printf("%.2f ",v[k].f32); + } + printf("\n"); + } + } + switch(operand_num) { + case 1 ://operand 1 + for(k=0;k<8;k++){ + mapping(thrd,LOAD_A,a_layout,F16_TYPE,k,16,row,col,offset); + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("A:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); + matrix_a[row][col]=nw_v[offset]; + } + break; + case 2 ://operand 2 + for(k=0;k<8;k++){ + mapping(thrd,LOAD_B,b_layout,F16_TYPE,k,16,row,col,offset); + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("B:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); + matrix_b[row][col]=nw_v[offset]; + } + break; + case 3 ://operand 3 + for(k=0;k<8;k++){ + mapping(thrd,LOAD_C,ROW,type2,k,16,row,col,offset); + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("C:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); + if(type2!=F16_TYPE){ + matrix_c[row][col]=v[offset]; + } + else { + matrix_c[row][col]=nw_v[offset]; + } + } + break; + default : + printf("Invalid Operand Index\n" ); + } + } + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("\n"); + } + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ + printf("MATRIX_A\n"); + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + temp=matrix_a[i][j].f16; + printf("%.2f ",temp); + } + printf("\n"); + } + printf("MATRIX_B\n"); + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + temp=matrix_b[i][j].f16; + printf("%.2f ",temp); + } + printf("\n"); + } + printf("MATRIX_C\n"); + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + if(type2==F16_TYPE){ + temp=matrix_c[i][j].f16; + printf("%.2f ",temp); + } + else + printf("%.2f ",matrix_c[i][j].f32); + } + printf("\n"); + } + } + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + matrix_d[i][j].f16=0; + } + } + + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + for(k=0;k<16;k++){ + matrix_d[i][j].f16=matrix_d[i][j].f16+matrix_a[i][k].f16*matrix_b[k][j].f16; + } + if((type==F16_TYPE)&&(type2==F16_TYPE)) + matrix_d[i][j].f16+=matrix_c[i][j].f16; + else if((type==F32_TYPE)&&(type2==F16_TYPE)){ + temp2=matrix_d[i][j].f16+matrix_c[i][j].f16; + temp=temp2; + matrix_d[i][j].f32=temp; + } + else if((type==F16_TYPE)&&(type2==F32_TYPE)){ + temp=matrix_d[i][j].f16; + temp+=matrix_c[i][j].f32; + matrix_d[i][j].f16=half(temp); + } + else{ + temp=matrix_d[i][j].f16; + temp+=matrix_c[i][j].f32; + matrix_d[i][j].f32=temp; + } + } + } + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ + printf("MATRIX_D\n"); + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + if(type==F16_TYPE){ + temp=matrix_d[i][j].f16; + printf("%.2f ",temp); + } + else + printf("%.2f ",matrix_d[i][j].f32); + } + printf("\n"); + } + } + for (thrd=0; thrd < core->get_warp_size(); thrd++){ + int row_t[8]; + int col_t[8]; + for(k=0;k<8;k++){ + mapping(thrd,LOAD_C,ROW,type,k,16,row_t[k],col_t[k],offset); + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma:store:row:%d,col%d\n",row_t[k],col_t[k]); + } + thread = core->get_thread_info()[tid+thrd]; + + + if(type==F32_TYPE){ + thread->set_wmma_vector_operand_values(dst,matrix_d[row_t[0]][col_t[0]],matrix_d[row_t[1]][col_t[1]],matrix_d[row_t[2]][col_t[2]],matrix_d[row_t[3]][col_t[3]],matrix_d[row_t[4]][col_t[4]],matrix_d[row_t[5]][col_t[5]],matrix_d[row_t[6]][col_t[6]],matrix_d[row_t[7]][col_t[7]]); + + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + { + printf("thread%d:",thrd); + for(k=0;k<8;k++){ + printf("%.2f ",matrix_d[row_t[k]][col_t[k]].f32); + } + printf("\n"); + } + } + else if(type==F16_TYPE){ + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ + printf("thread%d:",thrd); + for(k=0;k<8;k++){ + temp=matrix_d[row_t[k]][col_t[k]].f16; + printf("%.2f ",temp); + } + printf("\n"); - } else { - printf("wmma:mma:wrong type\n"); - abort(); - } - } + printf("thread%d:",thrd); + for(k=0;k<8;k++){ + printf("%x ", (unsigned int)matrix_d[row_t[k]][col_t[k]].f16); + } + printf("\n"); + } + ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4; + nw_data1.s64=((matrix_d[row_t[0]][col_t[0]].s64 & 0xffff))|((matrix_d[row_t[1]][col_t[1]].s64&0xffff)<<16); + nw_data2.s64=((matrix_d[row_t[2]][col_t[2]].s64 & 0xffff))|((matrix_d[row_t[3]][col_t[3]].s64&0xffff)<<16); + nw_data3.s64=((matrix_d[row_t[4]][col_t[4]].s64 & 0xffff))|((matrix_d[row_t[5]][col_t[5]].s64&0xffff)<<16); + nw_data4.s64=((matrix_d[row_t[6]][col_t[6]].s64 & 0xffff))|((matrix_d[row_t[7]][col_t[7]].s64&0xffff)<<16); + thread->set_vector_operand_values(dst,nw_data1,nw_data2,nw_data3,nw_data4); + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("thread%d=%llx,%llx,%llx,%llx", thrd, nw_data1.s64, nw_data2.s64, nw_data3.s64, nw_data4.s64); + + } + else{ + printf("wmma:mma:wrong type\n"); + abort(); + } + } } -void call_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - static unsigned call_uid_next = 1; +void call_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + static unsigned call_uid_next = 1; + + const operand_info &target = pI->func_addr(); + assert( target.is_function_address() ); + const symbol *func_addr = target.get_symbol(); + function_info *target_func = func_addr->get_pc(); + if (target_func->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", target_func->get_name().c_str() ); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", target_func->get_name().c_str() ); + /* + * Some of the instructions like printf() gives the gpgpusim the wrong impression that it is a function call. + * As printf() doesnt have a body like functions do, doing pdom analysis for printf() causes a crash. + */ + if (target_func->get_function_size() >0) + target_func->do_pdom(); + target_func->set_pdom(); + } - const operand_info &target = pI->func_addr(); - assert(target.is_function_address()); - const symbol *func_addr = target.get_symbol(); - function_info *target_func = func_addr->get_pc(); - if (target_func->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", - target_func->get_name().c_str()); - } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", - target_func->get_name().c_str()); - /* - * Some of the instructions like printf() gives the gpgpusim the wrong - * impression that it is a function call. - * As printf() doesnt have a body like functions do, doing pdom analysis for - * printf() causes a crash. - */ - if (target_func->get_function_size() > 0) target_func->do_pdom(); - target_func->set_pdom(); - } + // check that number of args and return match function requirements + if( pI->has_return() ^ target_func->has_return() ) { + printf("GPGPU-Sim PTX: Execution error - mismatch in number of return values between\n" + " call instruction and function declaration\n"); + abort(); + } + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + unsigned n_operands = pI->get_num_operands(); - // check that number of args and return match function requirements - if (pI->has_return() ^ target_func->has_return()) { - printf( - "GPGPU-Sim PTX: Execution error - mismatch in number of return values " - "between\n" - " call instruction and function declaration\n"); - abort(); - } - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - unsigned n_operands = pI->get_num_operands(); + if( n_operands != (n_return+1+n_args) ) { + printf("GPGPU-Sim PTX: Execution error - mismatch in number of arguements between\n" + " call instruction and function declaration\n"); + abort(); + } - if (n_operands != (n_return + 1 + n_args)) { - printf( - "GPGPU-Sim PTX: Execution error - mismatch in number of arguements " - "between\n" - " call instruction and function declaration\n"); - abort(); - } + // handle intrinsic functions + std::string fname = target_func->get_name(); + if( fname == "vprintf" ) { + gpgpusim_cuda_vprintf(pI, thread, target_func); + return; + } - // handle intrinsic functions - std::string fname = target_func->get_name(); - if (fname == "vprintf") { - gpgpusim_cuda_vprintf(pI, thread, target_func); - return; - } #if (CUDART_VERSION >= 5000) - // Jin: handle device runtime apis for CDP - else if (fname == "cudaGetParameterBufferV2") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_getParameterBufferV2( - pI, thread, target_func); - return; - } else if (fname == "cudaLaunchDeviceV2") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_launchDeviceV2( - pI, thread, target_func); - return; - } else if (fname == "cudaStreamCreateWithFlags") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_streamCreateWithFlags( - pI, thread, target_func); - return; - } + //Jin: handle device runtime apis for CDP + else if(fname == "cudaGetParameterBufferV2") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_getParameterBufferV2(pI, thread, target_func); + return; + } + else if(fname == "cudaLaunchDeviceV2") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_launchDeviceV2(pI, thread, target_func); + return; + } + else if(fname == "cudaStreamCreateWithFlags") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_streamCreateWithFlags(pI, thread, target_func); + return; + } #endif - // read source arguements into register specified in declaration of function - arg_buffer_list_t arg_values; - copy_args_into_buffer_list(pI, thread, target_func, arg_values); + // read source arguements into register specified in declaration of function + arg_buffer_list_t arg_values; + copy_args_into_buffer_list(pI, thread, target_func, arg_values); - // record local for return value (we only support a single return value) - const symbol *return_var_src = NULL; - const symbol *return_var_dst = NULL; - if (target_func->has_return()) { - return_var_dst = pI->dst().get_symbol(); - return_var_src = target_func->get_return_var(); - } + // record local for return value (we only support a single return value) + const symbol *return_var_src = NULL; + const symbol *return_var_dst = NULL; + if( target_func->has_return() ) { + return_var_dst = pI->dst().get_symbol(); + return_var_src = target_func->get_return_var(); + } - gpgpu_sim *gpu = thread->get_gpu(); - unsigned callee_pc = 0, callee_rpc = 0; - if (gpu->simd_model() == POST_DOMINATOR) { - thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(), - &callee_pc, &callee_rpc); - assert(callee_pc == thread->get_pc()); - } + gpgpu_sim *gpu = thread->get_gpu(); + unsigned callee_pc=0, callee_rpc=0; + if( gpu->simd_model() == POST_DOMINATOR ) { + thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(),&callee_pc,&callee_rpc); + assert( callee_pc == thread->get_pc() ); + } - thread->callstack_push(callee_pc + pI->inst_size(), callee_rpc, - return_var_src, return_var_dst, call_uid_next++); + thread->callstack_push(callee_pc + pI->inst_size(), callee_rpc, return_var_src, return_var_dst, call_uid_next++); - copy_buffer_list_into_frame(thread, arg_values); + copy_buffer_list_into_frame(thread, arg_values); - thread->set_npc(target_func); + thread->set_npc(target_func); } -// Ptxplus version of call instruction. Jumps to a label not a different Kernel. -void callp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - static unsigned call_uid_next = 1; +//Ptxplus version of call instruction. Jumps to a label not a different Kernel. +void callp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + + static unsigned call_uid_next = 1; - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = - thread->get_operand_value(target, target, U32_TYPE, thread, 1); + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); - const symbol *return_var_src = NULL; - const symbol *return_var_dst = NULL; + const symbol *return_var_src = NULL; + const symbol *return_var_dst = NULL; - gpgpu_sim *gpu = thread->get_gpu(); - unsigned callee_pc = 0, callee_rpc = 0; - if (gpu->simd_model() == POST_DOMINATOR) { - thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(), - &callee_pc, &callee_rpc); - assert(callee_pc == thread->get_pc()); - } + gpgpu_sim *gpu = thread->get_gpu(); + unsigned callee_pc=0, callee_rpc=0; + if( gpu->simd_model() == POST_DOMINATOR ) { + thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(),&callee_pc,&callee_rpc); + assert( callee_pc == thread->get_pc() ); + } - thread->callstack_push_plus(callee_pc + pI->inst_size(), callee_rpc, - return_var_src, return_var_dst, call_uid_next++); - thread->set_npc(target_pc); + thread->callstack_push_plus(callee_pc + pI->inst_size(), callee_rpc, return_var_src, return_var_dst, call_uid_next++); + thread->set_npc(target_pc); } -void clz_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void clz_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - int max; - unsigned long long mask; - d.u64 = 0; + int max; + unsigned long long mask; + d.u64 = 0; - switch (i_type) { - case B32_TYPE: + switch ( i_type ) { + case B32_TYPE: max = 32; mask = 0x80000000; break; - case B64_TYPE: + case B64_TYPE: max = 64; mask = 0x8000000000000000; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - while ((d.u32 < max) && ((a.u64 & mask) == 0)) { - d.u32++; - a.u64 = a.u64 << 1; - } + while ((d.u32 < max) && ((a.u64&mask) == 0) ) { + d.u32++; + a.u64 = a.u64 << 1; + } - thread->set_operand_value(dst, d, B32_TYPE, thread, pI); + thread->set_operand_value(dst,d, B32_TYPE, thread, pI); } -void cnot_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void cnot_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch (i_type) { - case PRED_TYPE: - d.pred = ((a.pred & 0x0001) == 0) ? 1 : 0; - break; - case B16_TYPE: - d.u16 = (a.u16 == 0) ? 1 : 0; - break; - case B32_TYPE: - d.u32 = (a.u32 == 0) ? 1 : 0; - break; - case B64_TYPE: - d.u64 = (a.u64 == 0) ? 1 : 0; - break; - default: + switch ( i_type ) { + case PRED_TYPE: d.pred = ((a.pred & 0x0001) == 0)?1:0; break; + case B16_TYPE: d.u16 = (a.u16 == 0)?1:0; break; + case B32_TYPE: d.u32 = (a.u32 == 0)?1:0; break; + case B64_TYPE: d.u64 = (a.u64 == 0)?1:0; break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -void cos_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void cos_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch (i_type) { - case F32_TYPE: + switch ( i_type ) { + case F32_TYPE: d.f32 = cos(a.f32); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -ptx_reg_t chop(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - switch (to_width) { - case 8: - x.mask_and(0, 0xFF); - break; - case 16: - x.mask_and(0, 0xFFFF); - break; - case 32: - x.mask_and(0, 0xFFFFFFFF); - break; - case 64: - break; - default: - assert(0); - } - return x; +ptx_reg_t chop( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + switch ( to_width ) { + case 8: x.mask_and(0,0xFF); break; + case 16: x.mask_and(0,0xFFFF); break; + case 32: x.mask_and(0,0xFFFFFFFF); break; + case 64: break; + default: assert(0); + } + return x; } -ptx_reg_t sext(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - x = chop(x, 0, from_width, 0, rounding_mode, saturation_mode); - switch (from_width) { - case 8: - if (x.get_bit(7)) x.mask_or(0xFFFFFFFF, 0xFFFFFF00); - break; - case 16: - if (x.get_bit(15)) x.mask_or(0xFFFFFFFF, 0xFFFF0000); - break; - case 32: - if (x.get_bit(31)) x.mask_or(0xFFFFFFFF, 0x00000000); - break; - case 64: - break; - default: - assert(0); - } - return x; +ptx_reg_t sext( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + x=chop(x,0,from_width,0,rounding_mode,saturation_mode); + switch ( from_width ) { + case 8: if ( x.get_bit(7) ) x.mask_or(0xFFFFFFFF,0xFFFFFF00);break; + case 16:if ( x.get_bit(15) ) x.mask_or(0xFFFFFFFF,0xFFFF0000);break; + case 32: if ( x.get_bit(31) ) x.mask_or(0xFFFFFFFF,0x00000000);break; + case 64: break; + default: assert(0); + } + return x; } -// sign extend depending on the destination register size - hack to get -// SobelFilter working in CUDA 4.2 -ptx_reg_t sexd(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - x = chop(x, 0, from_width, 0, rounding_mode, saturation_mode); - switch (to_width) { - case 8: - if (x.get_bit(7)) x.mask_or(0xFFFFFFFF, 0xFFFFFF00); - break; - case 16: - if (x.get_bit(15)) x.mask_or(0xFFFFFFFF, 0xFFFF0000); - break; - case 32: - if (x.get_bit(31)) x.mask_or(0xFFFFFFFF, 0x00000000); - break; - case 64: - break; - default: - assert(0); - } - return x; +// sign extend depending on the destination register size - hack to get SobelFilter working in CUDA 4.2 +ptx_reg_t sexd( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + x=chop(x,0,from_width,0,rounding_mode,saturation_mode); + switch ( to_width ) { + case 8: if ( x.get_bit(7) ) x.mask_or(0xFFFFFFFF,0xFFFFFF00);break; + case 16:if ( x.get_bit(15) ) x.mask_or(0xFFFFFFFF,0xFFFF0000);break; + case 32: if ( x.get_bit(31) ) x.mask_or(0xFFFFFFFF,0x00000000);break; + case 64: break; + default: assert(0); + } + return x; } -ptx_reg_t zext(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - return chop(x, 0, from_width, 0, rounding_mode, saturation_mode); +ptx_reg_t zext( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + return chop(x,0,from_width,0,rounding_mode,saturation_mode); } -int saturatei(int a, int max, int min) { - if (a > max) - a = max; - else if (a < min) - a = min; - return a; +int saturatei(int a, int max, int min) +{ + if (a > max) a = max; + else if (a < min) a = min; + return a; } -unsigned int saturatei(unsigned int a, unsigned int max) { - if (a > max) a = max; - return a; +unsigned int saturatei(unsigned int a, unsigned int max) +{ + if (a > max) a = max; + return a; } -ptx_reg_t f2x(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - half mytemp; - half_float::half tmp_h; - // assert( from_width == 32); +ptx_reg_t f2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + half mytemp; + half_float::half tmp_h; + //assert( from_width == 32); - enum cudaRoundMode mode = cudaRoundZero; - switch (rounding_mode) { - case RZI_OPTION: - mode = cudaRoundZero; - break; - case RNI_OPTION: - mode = cudaRoundNearest; - break; - case RMI_OPTION: - mode = cudaRoundMinInf; - break; - case RPI_OPTION: - mode = cudaRoundPosInf; - break; - default: - break; - } + enum cudaRoundMode mode = cudaRoundZero; + switch (rounding_mode) { + case RZI_OPTION: mode = cudaRoundZero; break; + case RNI_OPTION: mode = cudaRoundNearest; break; + case RMI_OPTION: mode = cudaRoundMinInf; break; + case RPI_OPTION: mode = cudaRoundPosInf; break; + default: break; + } - ptx_reg_t y; - if (to_sign == 1) { // convert to 64-bit number first? - int tmp = cuda_math::float2int(x.f32, mode); - if ((x.u32 & 0x7f800000) == 0) tmp = 0; // round denorm. FP to 0 - if (saturation_mode && to_width < 32) { - tmp = saturatei(tmp, (1 << to_width) - 1, -(1 << to_width)); - } - switch (to_width) { - case 8: - y.s8 = (char)tmp; - break; - case 16: - y.s16 = (short)tmp; - break; - case 32: - y.s32 = (int)tmp; - break; - case 64: - y.s64 = (long long)tmp; - break; - default: - assert(0); - break; - } - } else if (to_sign == 0) { - unsigned int tmp = cuda_math::float2uint(x.f32, mode); - if ((x.u32 & 0x7f800000) == 0) tmp = 0; // round denorm. FP to 0 - if (saturation_mode && to_width < 32) { - tmp = saturatei(tmp, (1 << to_width) - 1); - } - switch (to_width) { - case 8: - y.u8 = (unsigned char)tmp; - break; - case 16: - y.u16 = (unsigned short)tmp; - break; - case 32: - y.u32 = (unsigned int)tmp; - break; - case 64: - y.u64 = (unsigned long long)tmp; - break; - default: - assert(0); - break; - } - } else { - switch (to_width) { - case 16: - y.f16 = half_float::half_cast<half, - std::numeric_limits<float>::round_style>( - x.f32); // mytemp; - break; - case 32: - y.f32 = float(x.f16); - break; // handled by f2f - case 64: - y.f64 = x.f32; - break; - default: - assert(0); - break; - } - } - return y; + ptx_reg_t y; + if ( to_sign == 1 ) { // convert to 64-bit number first? + int tmp = cuda_math::float2int(x.f32, mode); + if ((x.u32 & 0x7f800000) == 0) + tmp = 0; // round denorm. FP to 0 + if (saturation_mode && to_width < 32) { + tmp = saturatei(tmp, (1<<to_width) - 1, -(1<<to_width)); + } + switch ( to_width ) { + case 8: y.s8 = (char)tmp; break; + case 16: y.s16 = (short)tmp; break; + case 32: y.s32 = (int)tmp; break; + case 64: y.s64 = (long long)tmp; break; + default: assert(0); break; + } + } else if ( to_sign == 0 ) { + unsigned int tmp = cuda_math::float2uint(x.f32, mode); + if ((x.u32 & 0x7f800000) == 0) + tmp = 0; // round denorm. FP to 0 + if (saturation_mode && to_width < 32) { + tmp = saturatei(tmp, (1<<to_width) - 1); + } + switch ( to_width ) { + case 8: y.u8 = (unsigned char)tmp; break; + case 16: y.u16 = (unsigned short)tmp; break; + case 32: y.u32 = (unsigned int)tmp; break; + case 64: y.u64 = (unsigned long long)tmp; break; + default: assert(0); break; + } + } else { + switch ( to_width ) { + case 16: + y.f16 =half_float::half_cast<half,std::numeric_limits<float>::round_style>(x.f32);//mytemp; + break; + case 32: + y.f32=float(x.f16); + break; // handled by f2f + case 64: + y.f64 = x.f32; + break; + default: assert(0); break; + } + } + return y; } -double saturated2i(double a, double max, double min) { - if (a > max) - a = max; - else if (a < min) - a = min; - return a; +double saturated2i (double a, double max, double min) { + if (a > max) a = max; + else if (a < min) a = min; + return a; } -ptx_reg_t d2x(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - assert(from_width == 64); +ptx_reg_t d2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + assert( from_width == 64); - double tmp; - switch (rounding_mode) { - case RZI_OPTION: - tmp = trunc(x.f64); - break; - case RNI_OPTION: - tmp = nearbyint(x.f64); - break; - case RMI_OPTION: - tmp = floor(x.f64); - break; - case RPI_OPTION: - tmp = ceil(x.f64); - break; - default: - tmp = x.f64; - break; - } + double tmp; + switch (rounding_mode) { + case RZI_OPTION: tmp = trunc(x.f64); break; + case RNI_OPTION: tmp = nearbyint(x.f64); break; + case RMI_OPTION: tmp = floor(x.f64); break; + case RPI_OPTION: tmp = ceil(x.f64); break; + default: tmp = x.f64; break; + } - ptx_reg_t y; - if (to_sign == 1) { - tmp = saturated2i(tmp, ((1 << (to_width - 1)) - 1), (1 << (to_width - 1))); - switch (to_width) { - case 8: - y.s8 = (char)tmp; - break; - case 16: - y.s16 = (short)tmp; - break; - case 32: - y.s32 = (int)tmp; - break; - case 64: - y.s64 = (long long)tmp; - break; - default: - assert(0); - break; - } - } else if (to_sign == 0) { - tmp = saturated2i(tmp, ((1 << (to_width - 1)) - 1), 0); - switch (to_width) { - case 8: - y.u8 = (unsigned char)tmp; - break; - case 16: - y.u16 = (unsigned short)tmp; - break; - case 32: - y.u32 = (unsigned int)tmp; - break; - case 64: - y.u64 = (unsigned long long)tmp; - break; - default: - assert(0); - break; - } - } else { - switch (to_width) { - case 16: - assert(0); - break; + ptx_reg_t y; + if ( to_sign == 1 ) { + tmp = saturated2i(tmp, ((1<<(to_width - 1)) - 1), (1<<(to_width - 1)) ); + switch ( to_width ) { + case 8: y.s8 = (char)tmp; break; + case 16: y.s16 = (short)tmp; break; + case 32: y.s32 = (int)tmp; break; + case 64: y.s64 = (long long)tmp; break; + default: assert(0); break; + } + } else if ( to_sign == 0 ) { + tmp = saturated2i(tmp, ((1<<(to_width - 1)) - 1), 0); + switch ( to_width ) { + case 8: y.u8 = (unsigned char)tmp; break; + case 16: y.u16 = (unsigned short)tmp; break; + case 32: y.u32 = (unsigned int)tmp; break; + case 64: y.u64 = (unsigned long long)tmp; break; + default: assert(0); break; + } + } else { + switch ( to_width ) { + case 16: assert(0); break; case 32: - y.f32 = x.f64; - break; - case 64: - y.f64 = x.f64; // should be handled by d2d - break; - default: - assert(0); - break; - } - } - return y; + y.f32 = x.f64; + break; + case 64: + y.f64 = x.f64; // should be handled by d2d + break; + default: assert(0); break; + } + } + return y; } -ptx_reg_t s2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - ptx_reg_t y; +ptx_reg_t s2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + ptx_reg_t y; - if (from_width < 64) { // 32-bit conversion - y = sext(x, from_width, 32, 0, rounding_mode, saturation_mode); + if (from_width < 64) { // 32-bit conversion + y = sext(x,from_width,32,0,rounding_mode,saturation_mode); - switch (to_width) { - case 16: - assert(0); - break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: - y.f32 = cuda_math::__int2float_rz(y.s32); - break; - case RN_OPTION: - y.f32 = cuda_math::__int2float_rn(y.s32); - break; - case RM_OPTION: - y.f32 = cuda_math::__int2float_rd(y.s32); - break; - case RP_OPTION: - y.f32 = cuda_math::__int2float_ru(y.s32); - break; - default: - break; - } - break; - case 64: - y.f64 = y.s32; - break; // no rounding needed - default: - assert(0); - break; - } - } else { - switch (to_width) { - case 16: - assert(0); - break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: - y.f32 = cuda_math::__ll2float_rz(y.s64); - break; - case RN_OPTION: - y.f32 = cuda_math::__ll2float_rn(y.s64); - break; - case RM_OPTION: - y.f32 = cuda_math::__ll2float_rd(y.s64); - break; - case RP_OPTION: - y.f32 = cuda_math::__ll2float_ru(y.s64); - break; - default: - break; - } - break; - case 64: - y.f64 = y.s64; - break; // no internal implementation found - default: - assert(0); - break; - } - } + switch ( to_width ) { + case 16: assert(0); break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: y.f32 = cuda_math::__int2float_rz(y.s32); break; + case RN_OPTION: y.f32 = cuda_math::__int2float_rn(y.s32); break; + case RM_OPTION: y.f32 = cuda_math::__int2float_rd(y.s32); break; + case RP_OPTION: y.f32 = cuda_math::__int2float_ru(y.s32); break; + default: break; + } + break; + case 64: y.f64 = y.s32; break; // no rounding needed + default: assert(0); break; + } + } else { + switch ( to_width ) { + case 16: assert(0); break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: y.f32 = cuda_math::__ll2float_rz(y.s64); break; + case RN_OPTION: y.f32 = cuda_math::__ll2float_rn(y.s64); break; + case RM_OPTION: y.f32 = cuda_math::__ll2float_rd(y.s64); break; + case RP_OPTION: y.f32 = cuda_math::__ll2float_ru(y.s64); break; + default: break; + } + break; + case 64: y.f64 = y.s64; break; // no internal implementation found + default: assert(0); break; + } + } - // saturating an integer to 1 or 0? - return y; + // saturating an integer to 1 or 0? + return y; } -ptx_reg_t u2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - ptx_reg_t y; +ptx_reg_t u2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + ptx_reg_t y; - if (from_width < 64) { // 32-bit conversion - y = zext(x, from_width, 32, 0, rounding_mode, saturation_mode); + if (from_width < 64) { // 32-bit conversion + y = zext(x,from_width,32,0,rounding_mode,saturation_mode); - switch (to_width) { - case 16: - assert(0); - break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: - y.f32 = cuda_math::__uint2float_rz(y.u32); - break; - case RN_OPTION: - y.f32 = cuda_math::__uint2float_rn(y.u32); - break; - case RM_OPTION: - y.f32 = cuda_math::__uint2float_rd(y.u32); - break; - case RP_OPTION: - y.f32 = cuda_math::__uint2float_ru(y.u32); - break; - default: - break; - } - break; - case 64: - y.f64 = y.u32; - break; // no rounding needed - default: - assert(0); - break; - } - } else { - switch (to_width) { - case 16: - assert(0); - break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: - y.f32 = cuda_math::__ull2float_rn(y.u64); - break; - case RN_OPTION: - y.f32 = cuda_math::__ull2float_rn(y.u64); - break; - case RM_OPTION: - y.f32 = cuda_math::__ull2float_rn(y.u64); - break; - case RP_OPTION: - y.f32 = cuda_math::__ull2float_rn(y.u64); - break; - default: - break; - } - break; - case 64: - y.f64 = y.u64; - break; // no internal implementation found - default: - assert(0); - break; - } - } + switch ( to_width ) { + case 16: assert(0); break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: y.f32 = cuda_math::__uint2float_rz(y.u32); break; + case RN_OPTION: y.f32 = cuda_math::__uint2float_rn(y.u32); break; + case RM_OPTION: y.f32 = cuda_math::__uint2float_rd(y.u32); break; + case RP_OPTION: y.f32 = cuda_math::__uint2float_ru(y.u32); break; + default: break; + } + break; + case 64: y.f64 = y.u32; break; // no rounding needed + default: assert(0); break; + } + } else { + switch ( to_width ) { + case 16: assert(0); break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; + case RN_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; + case RM_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; + case RP_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; + default: break; + } + break; + case 64: y.f64 = y.u64; break; // no internal implementation found + default: assert(0); break; + } + } - // saturating an integer to 1 or 0? - return y; + // saturating an integer to 1 or 0? + return y; } -ptx_reg_t f2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - ptx_reg_t y; - if (from_width == 16) { - half_float::detail::uint16 val = x.u16; - y.f32 = half_float::detail::half2float<float>(val); - } else { - switch (rounding_mode) { - case RZI_OPTION: - y.f32 = truncf(x.f32); - break; - case RNI_OPTION: -#if CUDART_VERSION >= 3000 - y.f32 = nearbyintf(x.f32); -#else - y.f32 = cuda_math::__internal_nearbyintf(x.f32); -#endif - break; - case RMI_OPTION: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = floorf(x.f32); - } - break; - case RPI_OPTION: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = ceilf(x.f32); - } - break; - default: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = x.f32; - } - break; - } -#if CUDART_VERSION >= 3000 - if (isnanf(y.f32)) -#else - if (cuda_math::__cuda___isnanf(y.f32)) -#endif - { - y.u32 = 0x7fffffff; - } else if (saturation_mode) { - y.f32 = cuda_math::__saturatef(y.f32); - } - } +ptx_reg_t f2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + ptx_reg_t y; + if (from_width == 16){ + half_float::detail::uint16 val = x.u16; + y.f32 = half_float::detail::half2float<float>(val); + }else{ + switch ( rounding_mode ) { + case RZI_OPTION: + y.f32 = truncf(x.f32); + break; + case RNI_OPTION: + #if CUDART_VERSION >= 3000 + y.f32 = nearbyintf(x.f32); + #else + y.f32 = cuda_math::__internal_nearbyintf(x.f32); + #endif + break; + case RMI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = floorf(x.f32); + } + break; + case RPI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = ceilf(x.f32); + } + break; + default: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = x.f32; + } + break; + } + #if CUDART_VERSION >= 3000 + if (isnanf(y.f32)) + #else + if (cuda_math::__cuda___isnanf(y.f32)) + #endif + { + y.u32 = 0x7fffffff; + } else if (saturation_mode) { + y.f32 = cuda_math::__saturatef(y.f32); + } + } - return y; + return y; } -ptx_reg_t d2d(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode) { - ptx_reg_t y; - switch (rounding_mode) { - case RZI_OPTION: - y.f64 = trunc(x.f64); - break; - case RNI_OPTION: +ptx_reg_t d2d( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + ptx_reg_t y; + switch ( rounding_mode ) { + case RZI_OPTION: + y.f64 = trunc(x.f64); + break; + case RNI_OPTION: #if CUDART_VERSION >= 3000 - y.f64 = nearbyint(x.f64); + y.f64 = nearbyint(x.f64); #else - y.f64 = cuda_math::__internal_nearbyintf(x.f64); + y.f64 = cuda_math::__internal_nearbyintf(x.f64); #endif - break; - case RMI_OPTION: - y.f64 = floor(x.f64); - break; - case RPI_OPTION: - y.f64 = ceil(x.f64); - break; - default: + break; + case RMI_OPTION: + y.f64 = floor(x.f64); + break; + case RPI_OPTION: + y.f64 = ceil(x.f64); + break; + default: y.f64 = x.f64; - break; - } - if (std::isnan(y.f64)) { - y.u64 = 0xfff8000000000000ull; - } else if (saturation_mode) { - y.f64 = cuda_math::__saturatef(y.f64); - } - return y; + break; + } + if (std::isnan(y.f64)) { + y.u64 = 0xfff8000000000000ull; + } else if (saturation_mode) { + y.f64 = cuda_math::__saturatef(y.f64); + } + return y; } -ptx_reg_t (*g_cvt_fn[11][11])(ptx_reg_t x, unsigned from_width, - unsigned to_width, int to_sign, int rounding_mode, - int saturation_mode) = { - {NULL, sext, sext, sext, NULL, sext, sext, sext, s2f, s2f, s2f}, - {chop, NULL, sext, sext, chop, NULL, sext, sext, s2f, s2f, s2f}, - {chop, sexd, NULL, sext, chop, chop, NULL, sext, s2f, s2f, s2f}, - {chop, chop, chop, NULL, chop, chop, chop, NULL, s2f, s2f, s2f}, - {NULL, zext, zext, zext, NULL, zext, zext, zext, u2f, u2f, u2f}, - {chop, NULL, zext, zext, chop, NULL, zext, zext, u2f, u2f, u2f}, - {chop, chop, NULL, zext, chop, chop, NULL, zext, u2f, u2f, u2f}, - {chop, chop, chop, NULL, chop, chop, chop, NULL, u2f, u2f, u2f}, - {f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, NULL, f2f, f2x}, - {f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2f, f2x}, - {d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2d}}; +ptx_reg_t (*g_cvt_fn[11][11])( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode ) = { + { NULL, sext, sext, sext, NULL, sext, sext, sext, s2f, s2f, s2f}, + { chop, NULL, sext, sext, chop, NULL, sext, sext, s2f, s2f, s2f}, + { chop, sexd, NULL, sext, chop, chop, NULL, sext, s2f, s2f, s2f}, + { chop, chop, chop, NULL, chop, chop, chop, NULL, s2f, s2f, s2f}, + { NULL, zext, zext, zext, NULL, zext, zext, zext, u2f, u2f, u2f}, + { chop, NULL, zext, zext, chop, NULL, zext, zext, u2f, u2f, u2f}, + { chop, chop, NULL, zext, chop, chop, NULL, zext, u2f, u2f, u2f}, + { chop, chop, chop, NULL, chop, chop, chop, NULL, u2f, u2f, u2f}, + { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , NULL,f2f, f2x}, + { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x, f2f, f2x}, + { d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x, d2x, d2d} +}; -void ptx_round(ptx_reg_t &data, int rounding_mode, int type) { - if (rounding_mode == RN_OPTION) { - return; - } - switch (rounding_mode) { - case RZI_OPTION: - switch (type) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); - assert(0); - break; - case F16_TYPE: - data.f16 = truncf(data.f16); - break; // assert(0); break; - case F32_TYPE: - data.f32 = truncf(data.f32); - break; - case F64_TYPE: - case FF64_TYPE: - if (data.f64 < 0) - data.f64 = ceil(data.f64); // negative - else - data.f64 = floor(data.f64); // positive - break; - default: - assert(0); - break; +void ptx_round(ptx_reg_t& data, int rounding_mode, int type) +{ + if (rounding_mode == RN_OPTION) { + return; + } + switch ( rounding_mode ) { + case RZI_OPTION: + switch ( type ) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); assert(0); break; + case F16_TYPE: data.f16=truncf(data.f16);break;//assert(0); break; + case F32_TYPE: + data.f32 = truncf(data.f32); + break; + case F64_TYPE: + case FF64_TYPE: + if (data.f64 < 0) data.f64 = ceil(data.f64); //negative + else data.f64 = floor(data.f64); //positive + break; + default: assert(0); break; } break; - case RNI_OPTION: - switch (type) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); - assert(0); - break; - case F16_TYPE: // assert(0); break; + case RNI_OPTION: + switch ( type ) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); assert(0); break; + case F16_TYPE:// assert(0); break; #if CUDART_VERSION >= 3000 - data.f16 = nearbyintf(data.f16); + data.f16 = nearbyintf(data.f16); #else - data.f16 = cuda_math::__cuda_nearbyintf(data.f16); + data.f16 = cuda_math::__cuda_nearbyintf(data.f16); #endif - break; - case F32_TYPE: + break; + case F32_TYPE: #if CUDART_VERSION >= 3000 - data.f32 = nearbyintf(data.f32); + data.f32 = nearbyintf(data.f32); #else - data.f32 = cuda_math::__cuda_nearbyintf(data.f32); + data.f32 = cuda_math::__cuda_nearbyintf(data.f32); #endif - break; - case F64_TYPE: - case FF64_TYPE: - data.f64 = round(data.f64); - break; - default: - assert(0); - break; + break; + case F64_TYPE: case FF64_TYPE: data.f64 = round(data.f64); break; + default: assert(0); break; } break; - case RMI_OPTION: - switch (type) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); - assert(0); - break; - case F16_TYPE: - data.f16 = floorf(data.f16); - break; // assert(0); break; - case F32_TYPE: - data.f32 = floorf(data.f32); - break; - case F64_TYPE: - case FF64_TYPE: - data.f64 = floor(data.f64); - break; - default: - assert(0); - break; + case RMI_OPTION: + switch ( type ) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); assert(0); break; + case F16_TYPE: data.f16=floorf(data.f16);break;//assert(0); break; + case F32_TYPE: + data.f32 = floorf(data.f32); + break; + case F64_TYPE: case FF64_TYPE: data.f64 = floor(data.f64); break; + default: assert(0); break; } break; - case RPI_OPTION: - switch (type) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); - assert(0); - break; - case F16_TYPE: - data.f16 = ceilf(data.f16); - break; // assert(0); break; - case F32_TYPE: - data.f32 = ceilf(data.f32); - break; - case F64_TYPE: - case FF64_TYPE: - data.f64 = ceil(data.f64); - break; - default: - assert(0); - break; + case RPI_OPTION: + switch ( type ) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); assert(0); break; + case F16_TYPE: data.f16 = ceilf(data.f16); break; //assert(0); break; + case F32_TYPE: data.f32 = ceilf(data.f32); break; + case F64_TYPE: case FF64_TYPE: data.f64 = ceil(data.f64); break; + default: assert(0); break; } break; - default: - break; - } + default: break; + } - if (type == F32_TYPE) { + if (type == F32_TYPE) { #if CUDART_VERSION >= 3000 - if (isnanf(data.f32)) + if (isnanf(data.f32)) #else - if (cuda_math::__cuda___isnanf(data.f32)) + if (cuda_math::__cuda___isnanf(data.f32)) #endif - { - data.u32 = 0x7fffffff; - } - } - if ((type == F64_TYPE) || (type == FF64_TYPE)) { - if (std::isnan(data.f64)) { - data.u64 = 0xfff8000000000000ull; - } - } + { + data.u32 = 0x7fffffff; + } + } + if ((type == F64_TYPE)||(type == FF64_TYPE)) { + if (std::isnan(data.f64)) { + data.u64 = 0xfff8000000000000ull; + } + } } -void ptx_saturate(ptx_reg_t &data, int saturation_mode, int type) { - if (!saturation_mode) { - return; - } - switch (type) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to clamp an integer to 1??\n"); - assert(0); - break; - case F16_TYPE: // assert(0); break; - if (data.f16 > 1.0f) data.f16 = 1.0f; // negative - if (data.f16 < 0.0f) data.f16 = 0.0f; // positive - break; - case F32_TYPE: - if (data.f32 > 1.0f) data.f32 = 1.0f; // negative - if (data.f32 < 0.0f) data.f32 = 0.0f; // positive - break; - case F64_TYPE: - case FF64_TYPE: - if (data.f64 > 1.0f) data.f64 = 1.0f; // negative - if (data.f64 < 0.0f) data.f64 = 0.0f; // positive - break; - default: - assert(0); - break; - } +void ptx_saturate(ptx_reg_t& data, int saturation_mode, int type) +{ + if (!saturation_mode) { + return; + } + switch ( type ) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to clamp an integer to 1??\n"); assert(0); break; + case F16_TYPE: //assert(0); break; + if (data.f16 > 1.0f) data.f16 = 1.0f; //negative + if (data.f16 < 0.0f) data.f16 = 0.0f; //positive + break; + case F32_TYPE: + if (data.f32 > 1.0f) data.f32 = 1.0f; //negative + if (data.f32 < 0.0f) data.f32 = 0.0f; //positive + break; + case F64_TYPE: + case FF64_TYPE: + if (data.f64 > 1.0f) data.f64 = 1.0f; //negative + if (data.f64 < 0.0f) data.f64 = 0.0f; //positive + break; + default: assert(0); break; + } + } -void cvt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned to_type = pI->get_type(); - unsigned from_type = pI->get_type2(); - unsigned rounding_mode = pI->rounding_mode(); - unsigned saturation_mode = pI->saturation_mode(); +void cvt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned to_type = pI->get_type(); + unsigned from_type = pI->get_type2(); + unsigned rounding_mode = pI->rounding_mode(); + unsigned saturation_mode = pI->saturation_mode(); + +// if ( to_type == F16_TYPE || from_type == F16_TYPE ) +// abort(); - // if ( to_type == F16_TYPE || from_type == F16_TYPE ) - // abort(); + int to_sign, from_sign; + size_t from_width, to_width; + unsigned src_fmt = type_info_key::type_decode(from_type, from_width, from_sign); + unsigned dst_fmt = type_info_key::type_decode(to_type, to_width, to_sign); - int to_sign, from_sign; - size_t from_width, to_width; - unsigned src_fmt = - type_info_key::type_decode(from_type, from_width, from_sign); - unsigned dst_fmt = type_info_key::type_decode(to_type, to_width, to_sign); + ptx_reg_t data = thread->get_operand_value(src1, dst, from_type, thread, 1); - ptx_reg_t data = thread->get_operand_value(src1, dst, from_type, thread, 1); + if(pI->is_neg()){ - if (pI->is_neg()) { - switch (from_type) { + switch( from_type ) { // Default to f32 for now, need to add support for others case S8_TYPE: case U8_TYPE: case B8_TYPE: - data.s8 = -data.s8; - break; + data.s8 = -data.s8; + break; case S16_TYPE: case U16_TYPE: case B16_TYPE: - data.s16 = -data.s16; - break; + data.s16 = -data.s16; + break; case S32_TYPE: case U32_TYPE: case B32_TYPE: - data.s32 = -data.s32; - break; + data.s32 = -data.s32; + break; case S64_TYPE: case U64_TYPE: case B64_TYPE: - data.s64 = -data.s64; - break; + data.s64 = -data.s64; + break; case F16_TYPE: - data.f16 = -data.f16; - break; + data.f16 = -data.f16; + break; case F32_TYPE: - data.f32 = -data.f32; - break; + data.f32 = -data.f32; + break; case F64_TYPE: case FF64_TYPE: - data.f64 = -data.f64; - break; + data.f64 = -data.f64; + break; default: - assert(0); - } - } + assert(0); + } - if (g_cvt_fn[src_fmt][dst_fmt] != NULL) { - ptx_reg_t result = g_cvt_fn[src_fmt][dst_fmt]( - data, from_width, to_width, to_sign, rounding_mode, saturation_mode); - data = result; - } + } - thread->set_operand_value(dst, data, to_type, thread, pI); -} -void cvta_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t data; + if ( g_cvt_fn[src_fmt][dst_fmt] != NULL ) { + ptx_reg_t result = g_cvt_fn[src_fmt][dst_fmt](data,from_width,to_width,to_sign, rounding_mode, saturation_mode); + data = result; + } - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - memory_space_t space = pI->get_space(); - bool to_non_generic = pI->is_to(); + thread->set_operand_value(dst, data, to_type, thread, pI ); +} - unsigned i_type = pI->get_type(); - ptx_reg_t from_addr = thread->get_operand_value(src1, dst, i_type, thread, 1); - addr_t from_addr_hw = (addr_t)from_addr.u64; - addr_t to_addr_hw = 0; - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); +void cvta_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t data; - if (to_non_generic) { - switch (space.get_type()) { - case shared_space: - to_addr_hw = generic_to_shared(smid, from_addr_hw); - break; - case local_space: - to_addr_hw = generic_to_local(smid, hwtid, from_addr_hw); - break; - case global_space: - to_addr_hw = generic_to_global(from_addr_hw); - break; - default: - abort(); - } - } else { - switch (space.get_type()) { - case shared_space: - to_addr_hw = shared_to_generic(smid, from_addr_hw); - break; - case local_space: - to_addr_hw = local_to_generic(smid, hwtid, from_addr_hw) + - thread->get_local_mem_stack_pointer(); - break; // add stack ptr here so that it can be passed as a pointer at - // function call - case global_space: - to_addr_hw = global_to_generic(from_addr_hw); - break; - default: - abort(); - } - } + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + memory_space_t space = pI->get_space(); + bool to_non_generic = pI->is_to(); + + unsigned i_type = pI->get_type(); + ptx_reg_t from_addr = thread->get_operand_value(src1,dst,i_type,thread,1); + addr_t from_addr_hw = (addr_t)from_addr.u64; + addr_t to_addr_hw = 0; + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); - ptx_reg_t to_addr; - to_addr.u64 = to_addr_hw; - thread->set_reg(dst.get_symbol(), to_addr); + if( to_non_generic ) { + switch( space.get_type() ) { + case shared_space: to_addr_hw = generic_to_shared( smid, from_addr_hw ); break; + case local_space: to_addr_hw = generic_to_local( smid, hwtid, from_addr_hw ); break; + case global_space: to_addr_hw = generic_to_global(from_addr_hw ); break; + default: abort(); + } + } else { + switch( space.get_type() ) { + case shared_space: to_addr_hw = shared_to_generic( smid, from_addr_hw ); break; + case local_space: to_addr_hw = local_to_generic( smid, hwtid, from_addr_hw ) + + thread->get_local_mem_stack_pointer(); break; // add stack ptr here so that it can be passed as a pointer at function call + case global_space: to_addr_hw = global_to_generic( from_addr_hw ); break; + default: abort(); + } + } + + ptx_reg_t to_addr; + to_addr.u64 = to_addr_hw; + thread->set_reg(dst.get_symbol(),to_addr); } -void div_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t data; +void div_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); + unsigned i_type = pI->get_type(); - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch (i_type) { - case S8_TYPE: - data.s8 = src1_data.s8 / src2_data.s8; - break; - case S16_TYPE: - data.s16 = src1_data.s16 / src2_data.s16; - break; - case S32_TYPE: - data.s32 = src1_data.s32 / src2_data.s32; - break; - case S64_TYPE: - data.s64 = src1_data.s64 / src2_data.s64; - break; - case U8_TYPE: - data.u8 = src1_data.u8 / src2_data.u8; - break; - case U16_TYPE: - data.u16 = src1_data.u16 / src2_data.u16; - break; - case U32_TYPE: - data.u32 = src1_data.u32 / src2_data.u32; - break; - case U64_TYPE: - data.u64 = src1_data.u64 / src2_data.u64; - break; - case B8_TYPE: - data.u8 = src1_data.u8 / src2_data.u8; - break; - case B16_TYPE: - data.u16 = src1_data.u16 / src2_data.u16; - break; - case B32_TYPE: - data.u32 = src1_data.u32 / src2_data.u32; - break; - case B64_TYPE: - data.u64 = src1_data.u64 / src2_data.u64; - break; - case F16_TYPE: - data.f16 = src1_data.f16 / src2_data.f16; - break; // assert(0); break; - case F32_TYPE: - data.f32 = src1_data.f32 / src2_data.f32; - break; - case F64_TYPE: - case FF64_TYPE: - data.f64 = src1_data.f64 / src2_data.f64; - break; - default: - assert(0); - break; - } - thread->set_operand_value(dst, data, i_type, thread, pI); + + switch ( i_type ) { + case S8_TYPE: + data.s8 = src1_data.s8 / src2_data.s8 ; break; + case S16_TYPE: + data.s16 = src1_data.s16 / src2_data.s16; break; + case S32_TYPE: + data.s32 = src1_data.s32 / src2_data.s32; break; + case S64_TYPE: + data.s64 = src1_data.s64 / src2_data.s64; break; + case U8_TYPE: + data.u8 = src1_data.u8 / src2_data.u8 ; break; + case U16_TYPE: + data.u16 = src1_data.u16 / src2_data.u16; break; + case U32_TYPE: + data.u32 = src1_data.u32 / src2_data.u32; break; + case U64_TYPE: + data.u64 = src1_data.u64 / src2_data.u64; break; + case B8_TYPE: + data.u8 = src1_data.u8 / src2_data.u8 ; break; + case B16_TYPE: + data.u16 = src1_data.u16 / src2_data.u16; break; + case B32_TYPE: + data.u32 = src1_data.u32 / src2_data.u32; break; + case B64_TYPE: + data.u64 = src1_data.u64 / src2_data.u64; break; + case F16_TYPE: data.f16 = src1_data.f16 / src2_data.f16; break;//assert(0); break; + case F32_TYPE: data.f32 = src1_data.f32 / src2_data.f32; break; + case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 / src2_data.f64; break; + default: assert(0); break; + } + thread->set_operand_value(dst,data, i_type, thread,pI); } -void dp4a_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - printf("DP4A instruction not implemented yet"); - assert(0); +void dp4a_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + printf("DP4A instruction not implemented yet"); + assert(0); + } -void ex2_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void ex2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); - unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch (i_type) { - case F32_TYPE: + switch ( i_type ) { + case F32_TYPE: data.f32 = cuda_math::__powf(2.0, src1_data.f32); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } - - thread->set_operand_value(dst, data, i_type, thread, pI); + } + + thread->set_operand_value(dst,data, i_type, thread,pI); } -void exit_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - thread->set_done(); - thread->exitCore(); - thread->registerExit(); +void exit_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + thread->set_done(); + thread->exitCore(); + thread->registerExit(); } -void mad_def(const ptx_instruction *pI, ptx_thread_info *thread, - bool use_carry = false); +void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry = false ); -void fma_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - mad_def(pI, thread); +void fma_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + mad_def(pI,thread); } -void isspacep_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a; - bool t = false; +void isspacep_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a; + bool t=false; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - memory_space_t space = pI->get_space(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + memory_space_t space = pI->get_space(); - a = thread->get_reg(src1.get_symbol()); - addr_t addr = (addr_t)a.u64; - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); + a = thread->get_reg(src1.get_symbol()); + addr_t addr = (addr_t)a.u64; + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); - switch (space.get_type()) { - case shared_space: - t = isspace_shared(smid, addr); - case local_space: - t = isspace_local(smid, hwtid, addr); - case global_space: - t = isspace_global(addr); - default: - abort(); - } + switch( space.get_type() ) { + case shared_space: t = isspace_shared( smid, addr ); + case local_space: t = isspace_local( smid, hwtid, addr ); + case global_space: t = isspace_global( addr ); + default: abort(); + } - ptx_reg_t p; - p.pred = t ? 1 : 0; + ptx_reg_t p; + p.pred = t?1:0; - thread->set_reg(dst.get_symbol(), p); + thread->set_reg(dst.get_symbol(),p); } -void decode_space(memory_space_t &space, ptx_thread_info *thread, - const operand_info &op, memory_space *&mem, addr_t &addr) { - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); +void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand_info &op, memory_space *&mem, addr_t &addr) +{ + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); - if (space == param_space_unclassified) { - // need to op to determine whether it refers to a kernel param or local - // param - const symbol *s = op.get_symbol(); - const type_info *t = s->type(); - type_info_key ti = t->get_key(); - if (ti.is_param_kernel()) - space = param_space_kernel; - else if (ti.is_param_local()) { - space = param_space_local; - } - // mov r1, param-label - else if (ti.is_reg()) { - space = param_space_kernel; - } else { - printf("GPGPU-Sim PTX: ERROR ** cannot resolve .param space for '%s'\n", - s->name().c_str()); - abort(); - } - } - switch (space.get_type()) { - case global_space: - mem = thread->get_global_memory(); - break; - case param_space_local: - case local_space: - mem = thread->m_local_mem; + if( space == param_space_unclassified ) { + // need to op to determine whether it refers to a kernel param or local param + const symbol *s = op.get_symbol(); + const type_info *t = s->type(); + type_info_key ti = t->get_key(); + if( ti.is_param_kernel() ) + space = param_space_kernel; + else if( ti.is_param_local() ) { + space = param_space_local; + } + //mov r1, param-label + else if (ti.is_reg() ){ + space = param_space_kernel; + } + else { + printf("GPGPU-Sim PTX: ERROR ** cannot resolve .param space for '%s'\n", s->name().c_str() ); + abort(); + } + } + switch ( space.get_type() ) { + case global_space: mem = thread->get_global_memory(); break; + case param_space_local: + case local_space: + mem = thread->m_local_mem; addr += thread->get_local_mem_stack_pointer(); - break; - case tex_space: - mem = thread->get_tex_memory(); - break; - case surf_space: - mem = thread->get_surf_memory(); - break; - case param_space_kernel: - mem = thread->get_param_memory(); - break; - case shared_space: - mem = thread->m_shared_mem; - break; - case sstarr_space: - mem = thread->m_sstarr_mem; - break; - case const_space: - mem = thread->get_global_memory(); - break; - case generic_space: - if (thread->get_ptx_version().ver() >= 2.0) { - // convert generic address to memory space address - space = whichspace(addr); - switch (space.get_type()) { - case global_space: - mem = thread->get_global_memory(); - addr = generic_to_global(addr); - break; - case local_space: - mem = thread->m_local_mem; - addr = generic_to_local(smid, hwtid, addr); - break; - case shared_space: - mem = thread->m_shared_mem; - addr = generic_to_shared(smid, addr); - break; - default: - abort(); - } + break; + case tex_space: mem = thread->get_tex_memory(); break; + case surf_space: mem = thread->get_surf_memory(); break; + case param_space_kernel: mem = thread->get_param_memory(); break; + case shared_space: mem = thread->m_shared_mem; break; + case sstarr_space: mem = thread->m_sstarr_mem; break; + case const_space: mem = thread->get_global_memory(); break; + case generic_space: + if( thread->get_ptx_version().ver() >= 2.0 ) { + // convert generic address to memory space address + space = whichspace(addr); + switch ( space.get_type() ) { + case global_space: mem = thread->get_global_memory(); addr = generic_to_global(addr); break; + case local_space: mem = thread->m_local_mem; addr = generic_to_local(smid,hwtid,addr); break; + case shared_space: mem = thread->m_shared_mem; addr = generic_to_shared(smid,addr); break; + default: abort(); + } } else { - abort(); + abort(); } break; - case param_space_unclassified: - case undefined_space: - default: + case param_space_unclassified: + case undefined_space: + default: abort(); - } + } } -void ld_exec(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void ld_exec( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned type = pI->get_type(); + unsigned type = pI->get_type(); - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, type, thread, 1); - ptx_reg_t data; - memory_space_t space = pI->get_space(); - unsigned vector_spec = pI->get_vector(); + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, type, thread, 1); + ptx_reg_t data; + memory_space_t space = pI->get_space(); + unsigned vector_spec = pI->get_vector(); - memory_space *mem = NULL; - addr_t addr = src1_data.u32; + memory_space *mem = NULL; + addr_t addr = src1_data.u32; - decode_space(space, thread, src1, mem, addr); + decode_space(space,thread,src1,mem,addr); - size_t size; - int t; - data.u64 = 0; - type_info_key::type_decode(type, size, t); - if (!vector_spec) { - mem->read(addr, size / 8, &data.s64); - if (type == S16_TYPE || type == S32_TYPE) sign_extend(data, size, dst); - thread->set_operand_value(dst, data, type, thread, pI); - } else { - ptx_reg_t data1, data2, data3, data4; - mem->read(addr, size / 8, &data1.s64); - mem->read(addr + size / 8, size / 8, &data2.s64); - if (vector_spec != V2_TYPE) { // either V3 or V4 - mem->read(addr + 2 * size / 8, size / 8, &data3.s64); - if (vector_spec != V3_TYPE) { // v4 - mem->read(addr + 3 * size / 8, size / 8, &data4.s64); - thread->set_vector_operand_values(dst, data1, data2, data3, data4); - } else // v3 - thread->set_vector_operand_values(dst, data1, data2, data3, data3); - } else // v2 - thread->set_vector_operand_values(dst, data1, data2, data2, data2); - } - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; + size_t size; + int t; + data.u64=0; + type_info_key::type_decode(type,size,t); + if (!vector_spec) { + mem->read(addr,size/8,&data.s64); + if( type == S16_TYPE || type == S32_TYPE ) + sign_extend(data,size,dst); + thread->set_operand_value(dst,data, type, thread, pI); + } else { + ptx_reg_t data1, data2, data3, data4; + mem->read(addr,size/8,&data1.s64); + mem->read(addr+size/8,size/8,&data2.s64); + if (vector_spec != V2_TYPE) { //either V3 or V4 + mem->read(addr+2*size/8,size/8,&data3.s64); + if (vector_spec != V3_TYPE) { //v4 + mem->read(addr+3*size/8,size/8,&data4.s64); + thread->set_vector_operand_values(dst,data1,data2,data3,data4); + } else //v3 + thread->set_vector_operand_values(dst,data1,data2,data3,data3); + } else //v2 + thread->set_vector_operand_values(dst,data1,data2,data2,data2); + } + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; } -void ld_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ld_exec(pI, thread); +void ld_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ld_exec(pI,thread); } -void ldu_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ld_exec(pI, thread); +void ldu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ld_exec(pI,thread); } -void mma_st_impl(const ptx_instruction *pI, core_t *core, warp_inst_t &inst) { - size_t size; - unsigned smid; - int t; - int thrd, k; - ptx_thread_info *thread; - - const operand_info &src = pI->operand_lookup(1); - const operand_info &src1 = pI->operand_lookup(0); - const operand_info &src2 = pI->operand_lookup(2); - int tid; - unsigned type = pI->get_type(); - unsigned wmma_type = pI->get_wmma_type(); - unsigned wmma_layout = pI->get_wmma_layout(0); - int stride; +void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t &inst ) +{ + size_t size; + unsigned smid; + int t; + int thrd, k; + ptx_thread_info *thread; + + const operand_info &src = pI->operand_lookup(1); + const operand_info &src1 = pI->operand_lookup(0); + const operand_info &src2 = pI->operand_lookup(2); + int tid ; + unsigned type = pI->get_type(); + unsigned wmma_type = pI->get_wmma_type(); + unsigned wmma_layout = pI->get_wmma_layout(0); + int stride; - if (core->get_gpu()->is_functional_sim()) - tid = inst.warp_id_func() * core->get_warp_size(); - else - tid = inst.warp_id() * core->get_warp_size(); + if(core->get_gpu()->is_functional_sim()) + tid= inst.warp_id_func()*core->get_warp_size(); + else + tid= inst.warp_id()*core->get_warp_size(); - _memory_op_t insn_memory_op = - pI->has_memory_read() ? memory_load : memory_store; - for (thrd = 0; thrd < core->get_warp_size(); thrd++) { - thread = core->get_thread_info()[tid + thrd]; + _memory_op_t insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; + for (thrd=0; thrd < core->get_warp_size(); thrd++) { + thread = core->get_thread_info()[tid+thrd]; ptx_reg_t addr_reg = thread->get_operand_value(src1, src, type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, src, type, thread, 1); - const operand_info &src_a = pI->operand_lookup(1); - unsigned nelem = src_a.get_vect_nelem(); - ptx_reg_t *v = new ptx_reg_t[8]; - thread->get_vector_operand_values(src_a, v, nelem); - stride = src2_data.u32; + ptx_reg_t src2_data = thread->get_operand_value(src2, src, type, thread, 1); + const operand_info &src_a= pI->operand_lookup(1); + unsigned nelem = src_a.get_vect_nelem(); + ptx_reg_t* v= new ptx_reg_t[8]; + thread->get_vector_operand_values( src_a, v, nelem ); + stride = src2_data.u32; + + memory_space_t space = pI->get_space(); - memory_space_t space = pI->get_space(); + memory_space *mem = NULL; + addr_t addr = addr_reg.u32; + + new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; + int num_mem_txn=0; + + smid = thread->get_hw_sid(); + if( whichspace(addr) == shared_space ) { + addr= generic_to_shared(smid,addr); + space = shared_space; + } + decode_space(space,thread,src1,mem,addr); - memory_space *mem = NULL; - addr_t addr = addr_reg.u32; + type_info_key::type_decode(type, size, t); + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma_st: thrd=%d, addr=%x, fp(size=%zu), stride=%d\n", thrd, addr_reg.u32, size, src2_data.u32); + addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8; + addr_t push_addr; - new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; - int num_mem_txn = 0; + ptx_reg_t nw_v[8]; + for(k=0;k<8;k++){ + if(k%2==0) + nw_v[k].s64=(v[k/2].s64&0xffff); + else + nw_v[k].s64=((v[k/2].s64&0xffff0000)>>16); + } - smid = thread->get_hw_sid(); - if (whichspace(addr) == shared_space) { - addr = generic_to_shared(smid, addr); - space = shared_space; - } - decode_space(space, thread, src1, mem, addr); + for(k=0;k<8;k++){ + if(type==F32_TYPE){ + //mem->write(new_addr+4*acc_float_offset(k,wmma_layout,stride),size/8,&v[k].s64,thread,pI); + push_addr=new_addr+4*acc_float_offset(k,wmma_layout,stride); + mem->write(push_addr,size/8,&v[k].s64,thread,pI); + mem_txn_addr[num_mem_txn++]=push_addr; + + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ + printf("wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n",thrd,v[0].s64,v[1].s64,v[2].s64,v[3].s64,v[4].s64,v[5].s64,v[6].s64,v[7].s64); + float temp; + int l; + printf("thread=%d:",thrd); + for(l=0;l<8;l++){ + temp=v[l].f32; + printf("%.2f",temp); + } + printf("\n"); + } + } + else if(type==F16_TYPE){ + if(wmma_layout==ROW){ + //mem->write(new_addr+k*2,size/8,&nw_v[k].s64,thread,pI); + push_addr=new_addr+k*2; + mem->write(push_addr,size/8,&nw_v[k].s64,thread,pI); + if(k%2==0) + mem_txn_addr[num_mem_txn++]=push_addr; + } + else if(wmma_layout==COL){ + //mem->write(new_addr+k*2*stride,size/8,&nw_v[k].s64,thread,pI); + push_addr=new_addr+k*2*stride; + mem->write(push_addr,size/8,&nw_v[k].s64,thread,pI); + mem_txn_addr[num_mem_txn++]=push_addr; + } + + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n",thrd,nw_v[0].s64,nw_v[1].s64,nw_v[2].s64,nw_v[3].s64,nw_v[4].s64,nw_v[5].s64,nw_v[6].s64,nw_v[7].s64); + } + } + + delete [] v; + inst.space = space; + inst.set_addr(thrd, (new_addr_type *)mem_txn_addr , num_mem_txn); - type_info_key::type_decode(type, size, t); - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma_st: thrd=%d, addr=%x, fp(size=%zu), stride=%d\n", thrd, - addr_reg.u32, size, src2_data.u32); - addr_t new_addr = - addr + - thread_group_offset(thrd, wmma_type, wmma_layout, type, stride) * size / - 8; - addr_t push_addr; + if((type==F16_TYPE)&&(wmma_layout==COL))//check the profiling xls for details + inst.data_size = 2; // 2 byte transaction + else + inst.data_size = 4; // 4 byte transaction - ptx_reg_t nw_v[8]; - for (k = 0; k < 8; k++) { - if (k % 2 == 0) - nw_v[k].s64 = (v[k / 2].s64 & 0xffff); - else - nw_v[k].s64 = ((v[k / 2].s64 & 0xffff0000) >> 16); - } - - for (k = 0; k < 8; k++) { - if (type == F32_TYPE) { - // mem->write(new_addr+4*acc_float_offset(k,wmma_layout,stride),size/8,&v[k].s64,thread,pI); - push_addr = new_addr + 4 * acc_float_offset(k, wmma_layout, stride); - mem->write(push_addr, size / 8, &v[k].s64, thread, pI); - mem_txn_addr[num_mem_txn++] = push_addr; - - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - printf( - "wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n", - thrd, v[0].s64, v[1].s64, v[2].s64, v[3].s64, v[4].s64, v[5].s64, - v[6].s64, v[7].s64); - float temp; - int l; - printf("thread=%d:", thrd); - for (l = 0; l < 8; l++) { - temp = v[l].f32; - printf("%.2f", temp); - } - printf("\n"); - } - } else if (type == F16_TYPE) { - if (wmma_layout == ROW) { - // mem->write(new_addr+k*2,size/8,&nw_v[k].s64,thread,pI); - push_addr = new_addr + k * 2; - mem->write(push_addr, size / 8, &nw_v[k].s64, thread, pI); - if (k % 2 == 0) mem_txn_addr[num_mem_txn++] = push_addr; - } else if (wmma_layout == COL) { - // mem->write(new_addr+k*2*stride,size/8,&nw_v[k].s64,thread,pI); - push_addr = new_addr + k * 2 * stride; - mem->write(push_addr, size / 8, &nw_v[k].s64, thread, pI); - mem_txn_addr[num_mem_txn++] = push_addr; - } - - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf( - "wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n", - thrd, nw_v[0].s64, nw_v[1].s64, nw_v[2].s64, nw_v[3].s64, - nw_v[4].s64, nw_v[5].s64, nw_v[6].s64, nw_v[7].s64); - } - } - - delete[] v; - inst.space = space; - inst.set_addr(thrd, (new_addr_type *)mem_txn_addr, num_mem_txn); - - if ((type == F16_TYPE) && - (wmma_layout == COL)) // check the profiling xls for details - inst.data_size = 2; // 2 byte transaction - else - inst.data_size = 4; // 4 byte transaction - - assert(inst.memory_op == insn_memory_op); - // thread->m_last_effective_address = addr; - // thread->m_last_memory_space = space; - } + assert( inst.memory_op == insn_memory_op ); + //thread->m_last_effective_address = addr; + //thread->m_last_memory_space = space; + } } -void mma_ld_impl(const ptx_instruction *pI, core_t *core, warp_inst_t &inst) { - size_t size; - int t, i; - unsigned smid; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t &inst ) +{ + size_t size; + int t,i; + unsigned smid; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned type = pI->get_type(); - unsigned wmma_type = pI->get_wmma_type(); - unsigned wmma_layout = pI->get_wmma_layout(0); - int tid; - int thrd, stride; - ptx_thread_info *thread; + unsigned type = pI->get_type(); + unsigned wmma_type = pI->get_wmma_type(); + unsigned wmma_layout = pI->get_wmma_layout(0); + int tid; + int thrd,stride; + ptx_thread_info *thread; + - if (core->get_gpu()->is_functional_sim()) - tid = inst.warp_id_func() * core->get_warp_size(); - else - tid = inst.warp_id() * core->get_warp_size(); + if(core->get_gpu()->is_functional_sim()) + tid= inst.warp_id_func()*core->get_warp_size(); + else + tid= inst.warp_id()*core->get_warp_size(); - _memory_op_t insn_memory_op = - pI->has_memory_read() ? memory_load : memory_store; + _memory_op_t insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; + + for (thrd=0; thrd < core->get_warp_size(); thrd++){ + thread = core->get_thread_info()[tid+thrd]; + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, U32_TYPE, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, dst, U32_TYPE, thread, 1); + stride=src2_data.u32; + memory_space_t space = pI->get_space(); - for (thrd = 0; thrd < core->get_warp_size(); thrd++) { - thread = core->get_thread_info()[tid + thrd]; - ptx_reg_t src1_data = - thread->get_operand_value(src1, dst, U32_TYPE, thread, 1); - ptx_reg_t src2_data = - thread->get_operand_value(src2, dst, U32_TYPE, thread, 1); - stride = src2_data.u32; - memory_space_t space = pI->get_space(); + memory_space *mem = NULL; + addr_t addr = src1_data.u32; + smid = thread->get_hw_sid(); + if( whichspace(addr) == shared_space ) { + addr= generic_to_shared(smid,addr); + space = shared_space; + } - memory_space *mem = NULL; - addr_t addr = src1_data.u32; - smid = thread->get_hw_sid(); - if (whichspace(addr) == shared_space) { - addr = generic_to_shared(smid, addr); - space = shared_space; - } + decode_space(space,thread,src1,mem,addr); + type_info_key::type_decode(type, size, t); + + ptx_reg_t data[16]; + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma_ld: thrd=%d,addr=%x, fpsize=%zu, stride=%d\n", thrd, src1_data.u32, size, src2_data.u32); + + addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8; + addr_t fetch_addr; + new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; + int num_mem_txn=0; - decode_space(space, thread, src1, mem, addr); - type_info_key::type_decode(type, size, t); + if(wmma_type==LOAD_A){ + for(i=0;i<16;i++){ + if(wmma_layout==ROW){ + //mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr=new_addr+2*i; + mem->read(fetch_addr,size/8,&data[i].s64); + } + else if(wmma_layout==COL){ + //mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); + fetch_addr=new_addr+2*(i%4)+2*stride*4*(i/4); + mem->read(fetch_addr,size/8,&data[i].s64); + } + else{ + printf("mma_ld:wrong_layout_type\n"); + abort(); + + } + if(i%2==0) + mem_txn_addr[num_mem_txn++]=fetch_addr; + } + } + else if(wmma_type==LOAD_B){ + for(i=0;i<16;i++){ + if(wmma_layout==COL){ + //mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr=new_addr+2*i; + mem->read(fetch_addr,size/8,&data[i].s64); + } + else if(wmma_layout==ROW){ + //mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); + fetch_addr=new_addr+2*(i%4)+2*stride*4*(i/4); + mem->read(fetch_addr,size/8,&data[i].s64); + } + else{ + printf("mma_ld:wrong_layout_type\n"); + abort(); + } + if(i%2==0) + mem_txn_addr[num_mem_txn++]=fetch_addr; + } + } + else if(wmma_type==LOAD_C){ + for(i=0;i<8;i++){ + if(type==F16_TYPE){ + if(wmma_layout==ROW){ + //mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr=new_addr+2*i; + mem->read(fetch_addr,size/8,&data[i].s64); + if(i%2==0) + mem_txn_addr[num_mem_txn++]=fetch_addr; + } + else if(wmma_layout==COL){ + //mem->read(new_addr+2*stride*i,size/8,&data[i].s64); + fetch_addr=new_addr+2*stride*i; + mem->read(fetch_addr,size/8,&data[i].s64); + mem_txn_addr[num_mem_txn++]=fetch_addr; + } + else{ + printf("mma_ld:wrong_type\n"); + abort(); + } + } + else if(type==F32_TYPE){ + //mem->read(new_addr+4*acc_float_offset(i,wmma_layout,stride),size/8,&data[i].s64); + fetch_addr=new_addr+4*acc_float_offset(i,wmma_layout,stride); + mem->read(fetch_addr,size/8,&data[i].s64); + mem_txn_addr[num_mem_txn++]=fetch_addr; + } + else{ + printf("wrong type"); + abort(); + } + } + } + else{ + printf("wrong wmma type\n");; + abort(); + } + //generate timing memory request + inst.space = space; + inst.set_addr(thrd, (new_addr_type *)mem_txn_addr , num_mem_txn); - ptx_reg_t data[16]; - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma_ld: thrd=%d,addr=%x, fpsize=%zu, stride=%d\n", thrd, - src1_data.u32, size, src2_data.u32); + if((wmma_type==LOAD_C)&&(type==F16_TYPE)&&(wmma_layout==COL))//memory address is scattered, check the profiling xls for more detail. + inst.data_size = 2; // 2 byte transaction + else + inst.data_size = 4; // 4 byte transaction + assert( inst.memory_op == insn_memory_op ); - addr_t new_addr = - addr + - thread_group_offset(thrd, wmma_type, wmma_layout, type, stride) * size / - 8; - addr_t fetch_addr; - new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; - int num_mem_txn = 0; + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ + if(type==F16_TYPE){ + printf("\nmma_ld:thread%d= ",thrd); + for(i=0;i<16;i++){ + printf("%llx ",data[i].u64); + } + printf("\n"); + + printf("\nmma_ld:thread%d= ",thrd); + float temp; + for(i=0;i<16;i++){ + temp=data[i].f16; + printf("%.2f ",temp); + } + printf("\n"); + } + else{ + printf("\nmma_ld:thread%d= ",thrd); + for(i=0;i<8;i++){ + printf("%.2f ",data[i].f32); + } + printf("\n"); + printf("\nmma_ld:thread%d= ",thrd); + for(i=0;i<8;i++){ + printf("%llx ",data[i].u64); + } + printf("\n"); + } + } - if (wmma_type == LOAD_A) { - for (i = 0; i < 16; i++) { - if (wmma_layout == ROW) { - // mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr = new_addr + 2 * i; - mem->read(fetch_addr, size / 8, &data[i].s64); - } else if (wmma_layout == COL) { - // mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); - fetch_addr = new_addr + 2 * (i % 4) + 2 * stride * 4 * (i / 4); - mem->read(fetch_addr, size / 8, &data[i].s64); - } else { - printf("mma_ld:wrong_layout_type\n"); - abort(); - } - if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; - } - } else if (wmma_type == LOAD_B) { - for (i = 0; i < 16; i++) { - if (wmma_layout == COL) { - // mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr = new_addr + 2 * i; - mem->read(fetch_addr, size / 8, &data[i].s64); - } else if (wmma_layout == ROW) { - // mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); - fetch_addr = new_addr + 2 * (i % 4) + 2 * stride * 4 * (i / 4); - mem->read(fetch_addr, size / 8, &data[i].s64); - } else { - printf("mma_ld:wrong_layout_type\n"); - abort(); - } - if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; - } - } else if (wmma_type == LOAD_C) { - for (i = 0; i < 8; i++) { - if (type == F16_TYPE) { - if (wmma_layout == ROW) { - // mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr = new_addr + 2 * i; - mem->read(fetch_addr, size / 8, &data[i].s64); - if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; - } else if (wmma_layout == COL) { - // mem->read(new_addr+2*stride*i,size/8,&data[i].s64); - fetch_addr = new_addr + 2 * stride * i; - mem->read(fetch_addr, size / 8, &data[i].s64); - mem_txn_addr[num_mem_txn++] = fetch_addr; - } else { - printf("mma_ld:wrong_type\n"); - abort(); - } - } else if (type == F32_TYPE) { - // mem->read(new_addr+4*acc_float_offset(i,wmma_layout,stride),size/8,&data[i].s64); - fetch_addr = new_addr + 4 * acc_float_offset(i, wmma_layout, stride); - mem->read(fetch_addr, size / 8, &data[i].s64); - mem_txn_addr[num_mem_txn++] = fetch_addr; - } else { - printf("wrong type"); - abort(); - } - } - } else { - printf("wrong wmma type\n"); - ; - abort(); - } - // generate timing memory request - inst.space = space; - inst.set_addr(thrd, (new_addr_type *)mem_txn_addr, num_mem_txn); - - if ((wmma_type == LOAD_C) && (type == F16_TYPE) && - (wmma_layout == COL)) // memory address is scattered, check the - // profiling xls for more detail. - inst.data_size = 2; // 2 byte transaction - else - inst.data_size = 4; // 4 byte transaction - assert(inst.memory_op == insn_memory_op); - - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - if (type == F16_TYPE) { - printf("\nmma_ld:thread%d= ", thrd); - for (i = 0; i < 16; i++) { - printf("%llx ", data[i].u64); - } - printf("\n"); - - printf("\nmma_ld:thread%d= ", thrd); - float temp; - for (i = 0; i < 16; i++) { - temp = data[i].f16; - printf("%.2f ", temp); - } - printf("\n"); - } else { - printf("\nmma_ld:thread%d= ", thrd); - for (i = 0; i < 8; i++) { - printf("%.2f ", data[i].f32); - } - printf("\n"); - printf("\nmma_ld:thread%d= ", thrd); - for (i = 0; i < 8; i++) { - printf("%llx ", data[i].u64); - } - printf("\n"); - } - } + if((wmma_type==LOAD_C)&&(type==F32_TYPE)){ + thread->set_wmma_vector_operand_values(dst,data[0],data[1],data[2],data[3],data[4],data[5],data[6],data[7]); + } + else{ + ptx_reg_t nw_data[8]; + int num_reg; + + if(wmma_type==LOAD_C) + num_reg=4; + else + num_reg=8; - if ((wmma_type == LOAD_C) && (type == F32_TYPE)) { - thread->set_wmma_vector_operand_values(dst, data[0], data[1], data[2], - data[3], data[4], data[5], data[6], - data[7]); - } else { - ptx_reg_t nw_data[8]; - int num_reg; + for(i=0;i<num_reg;i++){ + nw_data[i].s64= ((data[2*i].s64 & 0xffff)<<16)| ((data[2*i+1].s64 & 0xffff)); + } - if (wmma_type == LOAD_C) - num_reg = 4; - else - num_reg = 8; - - for (i = 0; i < num_reg; i++) { - nw_data[i].s64 = ((data[2 * i].s64 & 0xffff) << 16) | - ((data[2 * i + 1].s64 & 0xffff)); - } + if(wmma_type==LOAD_C) + thread->set_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3]); + else + thread->set_wmma_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3],nw_data[4],nw_data[5],nw_data[6],nw_data[7]); + if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ + printf("mma_ld:data[0].s64=%llx,data[1].s64=%llx,new_data[0].s64=%llx\n",data[0].u64,data[1].u64,nw_data[0].u64); + printf("mma_ld:data[2].s64=%llx,data[3].s64=%llx,new_data[1].s64=%llx\n",data[2].u64,data[3].u64,nw_data[1].u64); + printf("mma_ld:data[4].s64=%llx,data[5].s64=%llx,new_data[2].s64=%llx\n",data[4].u64,data[5].u64,nw_data[2].u64); + printf("mma_ld:data[6].s64=%llx,data[7].s64=%llx,new_data[3].s64=%llx\n",data[6].u64,data[7].u64,nw_data[3].u64); + if(wmma_type!=LOAD_C){ + printf("mma_ld:data[8].s64=%llx,data[9].s64=%llx,new_data[4].s64=%llx\n",data[8].u64,data[9].u64,nw_data[4].s64); + printf("mma_ld:data[10].s64=%llx,data[11].s64=%llx,new_data[5].s64=%llx\n",data[10].u64,data[11].u64,nw_data[5].u64); + printf("mma_ld:data[12].s64=%llx,data[13].s64=%llx,new_data[6].s64=%llx\n",data[12].u64,data[13].u64,nw_data[6].u64); + printf("mma_ld:data[14].s64=%llx,data[15].s64=%llx,new_data[7].s64=%llx\n",data[14].u64,data[15].u64,nw_data[3].u64); + } + } + } - if (wmma_type == LOAD_C) - thread->set_vector_operand_values(dst, nw_data[0], nw_data[1], - nw_data[2], nw_data[3]); - else - thread->set_wmma_vector_operand_values( - dst, nw_data[0], nw_data[1], nw_data[2], nw_data[3], nw_data[4], - nw_data[5], nw_data[6], nw_data[7]); - if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { - printf( - "mma_ld:data[0].s64=%llx,data[1].s64=%llx,new_data[0].s64=%llx\n", - data[0].u64, data[1].u64, nw_data[0].u64); - printf( - "mma_ld:data[2].s64=%llx,data[3].s64=%llx,new_data[1].s64=%llx\n", - data[2].u64, data[3].u64, nw_data[1].u64); - printf( - "mma_ld:data[4].s64=%llx,data[5].s64=%llx,new_data[2].s64=%llx\n", - data[4].u64, data[5].u64, nw_data[2].u64); - printf( - "mma_ld:data[6].s64=%llx,data[7].s64=%llx,new_data[3].s64=%llx\n", - data[6].u64, data[7].u64, nw_data[3].u64); - if (wmma_type != LOAD_C) { - printf( - "mma_ld:data[8].s64=%llx,data[9].s64=%llx,new_data[4].s64=%llx\n", - data[8].u64, data[9].u64, nw_data[4].s64); - printf( - "mma_ld:data[10].s64=%llx,data[11].s64=%llx,new_data[5].s64=%" - "llx\n", - data[10].u64, data[11].u64, nw_data[5].u64); - printf( - "mma_ld:data[12].s64=%llx,data[13].s64=%llx,new_data[6].s64=%" - "llx\n", - data[12].u64, data[13].u64, nw_data[6].u64); - printf( - "mma_ld:data[14].s64=%llx,data[15].s64=%llx,new_data[7].s64=%" - "llx\n", - data[14].u64, data[15].u64, nw_data[3].u64); - } - } - } - - // thread->m_last_effective_address = addr; - // thread->m_last_memory_space = space; - } + //thread->m_last_effective_address = addr; + //thread->m_last_memory_space = space; + } } -void lg2_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void lg2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); - unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch (i_type) { - case F32_TYPE: - d.f32 = log(a.f32) / log(2); + switch ( i_type ) { + case F32_TYPE: + d.f32 = log(a.f32)/log(2); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -void mad24_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t d, t; +void mad24_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t d, t; - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - unsigned sat_mode = pI->saturation_mode(); + unsigned sat_mode = pI->saturation_mode(); - assert(!pI->is_wide()); + assert( !pI->is_wide() ); - switch (i_type) { - case S32_TYPE: + switch ( i_type ) { + case S32_TYPE: t.s64 = a.s32 * b.s32; - if (pI->is_hi()) { - d.s64 = (t.s64 >> 16) + c.s32; - if (sat_mode) { - if (d.s64 > (int)0x7FFFFFFF) - d.s64 = (int)0x7FFFFFFF; - else if (d.s64 < (int)0x80000000) - d.s64 = (int)0x80000000; - } - } else if (pI->is_lo()) - d.s64 = t.s32 + c.s32; - else - assert(0); + if ( pI->is_hi() ) { + d.s64 = (t.s64>>16) + c.s32; + if ( sat_mode ) { + if ( d.s64 > (int)0x7FFFFFFF ) + d.s64 = (int)0x7FFFFFFF; + else if ( d.s64 < (int)0x80000000 ) + d.s64 = (int)0x80000000; + } + } else if ( pI->is_lo() ) d.s64 = t.s32 + c.s32; + else assert(0); break; - case U32_TYPE: + case U32_TYPE: t.u64 = a.u32 * b.u32; - if (pI->is_hi()) - d.u64 = (t.u64 >> 16) + c.u32; - else if (pI->is_lo()) - d.u64 = t.u32 + c.u32; - else - assert(0); + if ( pI->is_hi() ) d.u64 = (t.u64>>16) + c.u32; + else if ( pI->is_lo() ) d.u64 = t.u32 + c.u32; + else assert(0); break; - default: + default: assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void mad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - mad_def(pI, thread, false); +void mad_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + mad_def(pI, thread, false); } -void madp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - mad_def(pI, thread, true); +void madp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + mad_def(pI, thread, true); } -void madc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - mad_def(pI, thread, true); +void madc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + mad_def(pI, thread, true); } -void mad_def(const ptx_instruction *pI, ptx_thread_info *thread, - bool use_carry) { - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t d, t; +void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t d, t; - int carry = 0; - int overflow = 0; + int carry=0; + int overflow=0; - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - // take the carry bit, it should be the 4th operand - ptx_reg_t carry_bit; - carry_bit.u64 = 0; - if (use_carry) { - const operand_info &carry = pI->operand_lookup(4); - carry_bit = thread->get_operand_value(carry, dst, PRED_TYPE, thread, 0); - carry_bit.pred &= 0x4; - carry_bit.pred >>= 2; - } + // take the carry bit, it should be the 4th operand + ptx_reg_t carry_bit; + carry_bit.u64 = 0; + if (use_carry) { + const operand_info &carry = pI->operand_lookup(4); + carry_bit = thread->get_operand_value(carry, dst, PRED_TYPE, thread, 0); + carry_bit.pred &= 0x4; + carry_bit.pred >>=2; + } - unsigned rounding_mode = pI->rounding_mode(); + unsigned rounding_mode = pI->rounding_mode(); - switch (i_type) { - case S16_TYPE: + switch ( i_type ) { + case S16_TYPE: t.s32 = a.s16 * b.s16; - if (pI->is_wide()) - d.s32 = t.s32 + c.s32 + carry_bit.pred; - else if (pI->is_hi()) - d.s16 = (t.s32 >> 16) + c.s16 + carry_bit.pred; - else if (pI->is_lo()) - d.s16 = t.s16 + c.s16 + carry_bit.pred; - else - assert(0); - carry = - ((long long int)(t.s32 + c.s32 + carry_bit.pred) & 0x100000000) >> 32; + if ( pI->is_wide() ) d.s32 = t.s32 + c.s32 + carry_bit.pred; + else if ( pI->is_hi() ) d.s16 = (t.s32>>16) + c.s16 + carry_bit.pred; + else if ( pI->is_lo() ) d.s16 = t.s16 + c.s16 + carry_bit.pred; + else assert(0); + carry = ((long long int)(t.s32 + c.s32 + carry_bit.pred)&0x100000000)>>32; break; - case S32_TYPE: + case S32_TYPE: t.s64 = a.s32 * b.s32; - if (pI->is_wide()) - d.s64 = t.s64 + c.s64 + carry_bit.pred; - else if (pI->is_hi()) - d.s32 = (t.s64 >> 32) + c.s32 + carry_bit.pred; - else if (pI->is_lo()) - d.s32 = t.s32 + c.s32 + carry_bit.pred; - else - assert(0); + if ( pI->is_wide() ) d.s64 = t.s64 + c.s64 + carry_bit.pred; + else if ( pI->is_hi() ) d.s32 = (t.s64>>32) + c.s32 + carry_bit.pred; + else if ( pI->is_lo() ) d.s32 = t.s32 + c.s32 + carry_bit.pred; + else assert(0); break; - case S64_TYPE: + case S64_TYPE: t.s64 = a.s64 * b.s64; - assert(!pI->is_wide()); - assert(!pI->is_hi()); - assert(use_carry == false); - if (pI->is_lo()) - d.s64 = t.s64 + c.s64 + carry_bit.pred; - else - assert(0); + assert( !pI->is_wide() ); + assert( !pI->is_hi() ); + assert( use_carry == false); + if ( pI->is_lo() ) d.s64 = t.s64 + c.s64 + carry_bit.pred; + else assert(0); break; - case U16_TYPE: + case U16_TYPE: t.u32 = a.u16 * b.u16; - if (pI->is_wide()) - d.u32 = t.u32 + c.u32 + carry_bit.pred; - else if (pI->is_hi()) - d.u16 = (t.u32 + c.u16 + carry_bit.pred) >> 16; - else if (pI->is_lo()) - d.u16 = t.u16 + c.u16 + carry_bit.pred; - else - assert(0); - carry = ((long long int)((long long int)t.u32 + c.u32 + carry_bit.pred) & - 0x100000000) >> - 32; + if ( pI->is_wide() ) d.u32 = t.u32 + c.u32 + carry_bit.pred; + else if ( pI->is_hi() ) d.u16 = (t.u32 + c.u16 + carry_bit.pred)>>16; + else if ( pI->is_lo() ) d.u16 = t.u16 + c.u16 + carry_bit.pred; + else assert(0); + carry = ((long long int)((long long int)t.u32 + c.u32 + carry_bit.pred)&0x100000000)>>32; break; - case U32_TYPE: + case U32_TYPE: t.u64 = a.u32 * b.u32; - if (pI->is_wide()) - d.u64 = t.u64 + c.u64 + carry_bit.pred; - else if (pI->is_hi()) - d.u32 = (t.u64 + c.u32 + carry_bit.pred) >> 32; - else if (pI->is_lo()) - d.u32 = t.u32 + c.u32 + carry_bit.pred; - else - assert(0); + if ( pI->is_wide() ) d.u64 = t.u64 + c.u64 + carry_bit.pred; + else if ( pI->is_hi() ) d.u32 = (t.u64 + c.u32 + carry_bit.pred)>>32; + else if ( pI->is_lo() ) d.u32 = t.u32 + c.u32 + carry_bit.pred; + else assert(0); break; - case U64_TYPE: + case U64_TYPE: t.u64 = a.u64 * b.u64; - assert(!pI->is_wide()); - assert(!pI->is_hi()); - assert(use_carry == false); - if (pI->is_lo()) - d.u64 = t.u64 + c.u64 + carry_bit.pred; - else - assert(0); + assert( !pI->is_wide() ); + assert( !pI->is_hi() ); + assert( use_carry == false); + if ( pI->is_lo() ) d.u64 = t.u64 + c.u64 + carry_bit.pred; + else assert(0); break; - case F16_TYPE: { - // assert(0); - // break; - assert(use_carry == false); - int orig_rm = fegetround(); - switch (rounding_mode) { - case RN_OPTION: - break; - case RZ_OPTION: - fesetround(FE_TOWARDZERO); - break; - default: - assert(0); - break; - } - d.f16 = a.f16 * b.f16 + c.f16; - if (pI->saturation_mode()) { - if (d.f16 < 0) - d.f16 = 0; - else if (d.f16 > 1.0f) - d.f16 = 1.0f; - } - fesetround(orig_rm); - break; - } - case F32_TYPE: { - assert(use_carry == false); - int orig_rm = fegetround(); - switch (rounding_mode) { - case RN_OPTION: - break; - case RZ_OPTION: - fesetround(FE_TOWARDZERO); - break; - default: - assert(0); - break; - } - d.f32 = a.f32 * b.f32 + c.f32; - if (pI->saturation_mode()) { - if (d.f32 < 0) - d.f32 = 0; - else if (d.f32 > 1.0f) - d.f32 = 1.0f; - } - fesetround(orig_rm); - break; - } - case F64_TYPE: - case FF64_TYPE: { - assert(use_carry == false); - int orig_rm = fegetround(); - switch (rounding_mode) { - case RN_OPTION: - break; - case RZ_OPTION: - fesetround(FE_TOWARDZERO); - break; - default: - assert(0); - break; + case F16_TYPE:{ + // assert(0); + // break; + assert( use_carry == false); + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + d.f16 = a.f16 * b.f16 + c.f16; + if ( pI->saturation_mode() ) { + if ( d.f16 < 0 ) d.f16 = 0; + else if ( d.f16 > 1.0f ) d.f16 = 1.0f; + } + fesetround( orig_rm ); + break; + } + case F32_TYPE: { + assert( use_carry == false); + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + d.f32 = a.f32 * b.f32 + c.f32; + if ( pI->saturation_mode() ) { + if ( d.f32 < 0 ) d.f32 = 0; + else if ( d.f32 > 1.0f ) d.f32 = 1.0f; + } + fesetround( orig_rm ); + break; + } + case F64_TYPE: case FF64_TYPE: { + assert( use_carry == false); + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + d.f64 = a.f64 * b.f64 + c.f64; + if ( pI->saturation_mode() ) { + if ( d.f64 < 0 ) d.f64 = 0; + else if ( d.f64 > 1.0f ) d.f64 = 1.0; + } + fesetround( orig_rm ); + break; } - d.f64 = a.f64 * b.f64 + c.f64; - if (pI->saturation_mode()) { - if (d.f64 < 0) - d.f64 = 0; - else if (d.f64 > 1.0f) - d.f64 = 1.0; - } - fesetround(orig_rm); - break; - } - default: + default: assert(0); break; - } - thread->set_operand_value(dst, d, i_type, thread, pI, overflow, carry); + } + thread->set_operand_value(dst, d, i_type, thread, pI, overflow, carry); } -bool isNaN(float x) { return std::isnan(x); } +bool isNaN(float x) +{ + return std::isnan(x); +} -bool isNaN(double x) { return std::isnan(x); } +bool isNaN(double x) +{ + return std::isnan(x); +} -void max_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void max_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch (i_type) { - case U16_TYPE: - d.u16 = MY_MAX_I(a.u16, b.u16); - break; - case U32_TYPE: - d.u32 = MY_MAX_I(a.u32, b.u32); - break; - case U64_TYPE: - d.u64 = MY_MAX_I(a.u64, b.u64); - break; - case S16_TYPE: - d.s16 = MY_MAX_I(a.s16, b.s16); - break; - case S32_TYPE: - d.s32 = MY_MAX_I(a.s32, b.s32); - break; - case S64_TYPE: - d.s64 = MY_MAX_I(a.s64, b.s64); - break; - case F32_TYPE: - d.f32 = MY_MAX_F(a.f32, b.f32); - break; - case F64_TYPE: - case FF64_TYPE: - d.f64 = MY_MAX_F(a.f64, b.f64); - break; - default: + + switch ( i_type ) { + case U16_TYPE: d.u16 = MY_MAX_I(a.u16,b.u16); break; + case U32_TYPE: d.u32 = MY_MAX_I(a.u32,b.u32); break; + case U64_TYPE: d.u64 = MY_MAX_I(a.u64,b.u64); break; + case S16_TYPE: d.s16 = MY_MAX_I(a.s16,b.s16); break; + case S32_TYPE: d.s32 = MY_MAX_I(a.s32,b.s32); break; + case S64_TYPE: d.s64 = MY_MAX_I(a.s64,b.s64); break; + case F32_TYPE: d.f32 = MY_MAX_F(a.f32,b.f32); break; + case F64_TYPE: case FF64_TYPE: d.f64 = MY_MAX_F(a.f64,b.f64); break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -void membar_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - // handled by timing simulator +void membar_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + // handled by timing simulator } -void min_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void min_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch (i_type) { - case U16_TYPE: - d.u16 = MY_MIN_I(a.u16, b.u16); - break; - case U32_TYPE: - d.u32 = MY_MIN_I(a.u32, b.u32); - break; - case U64_TYPE: - d.u64 = MY_MIN_I(a.u64, b.u64); - break; - case S16_TYPE: - d.s16 = MY_MIN_I(a.s16, b.s16); - break; - case S32_TYPE: - d.s32 = MY_MIN_I(a.s32, b.s32); - break; - case S64_TYPE: - d.s64 = MY_MIN_I(a.s64, b.s64); - break; - case F32_TYPE: - d.f32 = MY_MIN_F(a.f32, b.f32); - break; - case F64_TYPE: - case FF64_TYPE: - d.f64 = MY_MIN_F(a.f64, b.f64); - break; - default: + + switch ( i_type ) { + case U16_TYPE: d.u16 = MY_MIN_I(a.u16,b.u16); break; + case U32_TYPE: d.u32 = MY_MIN_I(a.u32,b.u32); break; + case U64_TYPE: d.u64 = MY_MIN_I(a.u64,b.u64); break; + case S16_TYPE: d.s16 = MY_MIN_I(a.s16,b.s16); break; + case S32_TYPE: d.s32 = MY_MIN_I(a.s32,b.s32); break; + case S64_TYPE: d.s64 = MY_MIN_I(a.s64,b.s64); break; + case F32_TYPE: d.f32 = MY_MIN_F(a.f32,b.f32); break; + case F64_TYPE: case FF64_TYPE: d.f64 = MY_MIN_F(a.f64,b.f64); break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -void mov_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t data; +void mov_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - assert(src1.is_param_local() == 0); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + assert( src1.is_param_local() == 0 ); - if ((src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && - (i_type != BB128_TYPE) && (i_type != FF64_TYPE)) { - // pack or unpack operation - unsigned nbits_to_move; - ptx_reg_t tmp_bits; + if( (src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && (i_type != BB128_TYPE) && (i_type != FF64_TYPE) ) { + // pack or unpack operation + unsigned nbits_to_move; + ptx_reg_t tmp_bits; - switch (pI->get_type()) { - case B16_TYPE: - nbits_to_move = 16; - break; - case B32_TYPE: - nbits_to_move = 32; - break; - case B64_TYPE: - nbits_to_move = 64; - break; - default: - printf( - "Execution error: mov pack/unpack with unsupported type " - "qualifier\n"); - assert(0); - break; - } + switch( pI->get_type() ) { + case B16_TYPE: nbits_to_move = 16; break; + case B32_TYPE: nbits_to_move = 32; break; + case B64_TYPE: nbits_to_move = 64; break; + default: printf("Execution error: mov pack/unpack with unsupported type qualifier\n"); assert(0); break; + } - if (src1.is_vector()) { - unsigned nelem = src1.get_vect_nelem(); - ptx_reg_t v[4]; - thread->get_vector_operand_values(src1, v, nelem); + if( src1.is_vector() ) { + unsigned nelem = src1.get_vect_nelem(); + ptx_reg_t v[4]; + thread->get_vector_operand_values(src1, v, nelem ); - unsigned bits_per_src_elem = nbits_to_move / nelem; - for (unsigned i = 0; i < nelem; i++) { - switch (bits_per_src_elem) { - case 8: - tmp_bits.u64 |= ((unsigned long long)(v[i].u8) << (8 * i)); - break; - case 16: - tmp_bits.u64 |= ((unsigned long long)(v[i].u16) << (16 * i)); - break; - case 32: - tmp_bits.u64 |= ((unsigned long long)(v[i].u32) << (32 * i)); - break; - default: - printf( - "Execution error: mov pack/unpack with unsupported source/dst " - "size ratio (src)\n"); - assert(0); - break; - } - } - } else { - data = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned bits_per_src_elem = nbits_to_move / nelem; + for( unsigned i=0; i < nelem; i++ ) { + switch(bits_per_src_elem) { + case 8: tmp_bits.u64 |= ((unsigned long long)(v[i].u8) << (8*i)); break; + case 16: tmp_bits.u64 |= ((unsigned long long)(v[i].u16) << (16*i)); break; + case 32: tmp_bits.u64 |= ((unsigned long long)(v[i].u32) << (32*i)); break; + default: printf("Execution error: mov pack/unpack with unsupported source/dst size ratio (src)\n"); assert(0); break; + } + } + } else { + data = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch (pI->get_type()) { - case B16_TYPE: - tmp_bits.u16 = data.u16; - break; - case B32_TYPE: - tmp_bits.u32 = data.u32; - break; - case B64_TYPE: - tmp_bits.u64 = data.u64; - break; - default: - assert(0); - break; + switch( pI->get_type() ) { + case B16_TYPE: tmp_bits.u16 = data.u16; break; + case B32_TYPE: tmp_bits.u32 = data.u32; break; + case B64_TYPE: tmp_bits.u64 = data.u64; break; + default: assert(0); break; + } } - } - if (dst.is_vector()) { - unsigned nelem = dst.get_vect_nelem(); - ptx_reg_t v[4]; - unsigned bits_per_dst_elem = nbits_to_move / nelem; - for (unsigned i = 0; i < nelem; i++) { - switch (bits_per_dst_elem) { - case 8: - v[i].u8 = (tmp_bits.u64 >> (8 * i)) & ((unsigned long long)0xFF); - break; - case 16: - v[i].u16 = - (tmp_bits.u64 >> (16 * i)) & ((unsigned long long)0xFFFF); - break; - case 32: - v[i].u32 = - (tmp_bits.u64 >> (32 * i)) & ((unsigned long long)0xFFFFFFFF); - break; - default: - printf( - "Execution error: mov pack/unpack with unsupported source/dst " - "size ratio (dst)\n"); - assert(0); - break; - } + if( dst.is_vector() ) { + unsigned nelem = dst.get_vect_nelem(); + ptx_reg_t v[4]; + unsigned bits_per_dst_elem = nbits_to_move / nelem; + for( unsigned i=0; i < nelem; i++ ) { + switch(bits_per_dst_elem) { + case 8: v[i].u8 = (tmp_bits.u64 >> (8*i)) & ((unsigned long long) 0xFF); break; + case 16: v[i].u16 = (tmp_bits.u64 >> (16*i)) & ((unsigned long long) 0xFFFF); break; + case 32: v[i].u32 = (tmp_bits.u64 >> (32*i)) & ((unsigned long long) 0xFFFFFFFF); break; + default: + printf("Execution error: mov pack/unpack with unsupported source/dst size ratio (dst)\n"); + assert(0); + break; + } + } + thread->set_vector_operand_values(dst,v[0],v[1],v[2],v[3]); + } else { + thread->set_operand_value(dst,tmp_bits, i_type, thread, pI); } - thread->set_vector_operand_values(dst, v[0], v[1], v[2], v[3]); - } else { - thread->set_operand_value(dst, tmp_bits, i_type, thread, pI); - } - } else if (i_type == PRED_TYPE and src1.is_literal() == true) { - // in ptx, literal input translate to predicate as 0 = false and 1 = true - // we have adopted the opposite to simplify implementation of zero flags in - // ptxplus - data = thread->get_operand_value(src1, dst, i_type, thread, 1); + } else if (i_type == PRED_TYPE and src1.is_literal() == true) { + // in ptx, literal input translate to predicate as 0 = false and 1 = true + // we have adopted the opposite to simplify implementation of zero flags in ptxplus + data = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t finaldata; - finaldata.pred = (data.u32 == 0) ? 1 : 0; // setting zero-flag in predicate - thread->set_operand_value(dst, finaldata, i_type, thread, pI); - } else { - data = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t finaldata; + finaldata.pred = (data.u32 == 0)? 1 : 0; // setting zero-flag in predicate + thread->set_operand_value(dst, finaldata, i_type, thread, pI); + } else { - thread->set_operand_value(dst, data, i_type, thread, pI); - } + data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + thread->set_operand_value(dst, data, i_type, thread, pI); + + } } -void mul24_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; +void mul24_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - // src1_data = srcOperandModifiers(src1_data, src1, dst, i_type, thread); - // src2_data = srcOperandModifiers(src2_data, src2, dst, i_type, thread); + //src1_data = srcOperandModifiers(src1_data, src1, dst, i_type, thread); + //src2_data = srcOperandModifiers(src2_data, src2, dst, i_type, thread); - src1_data.mask_and(0, 0x00FFFFFF); - src2_data.mask_and(0, 0x00FFFFFF); + src1_data.mask_and(0,0x00FFFFFF); + src2_data.mask_and(0,0x00FFFFFF); - switch (i_type) { - case S32_TYPE: - if (src1_data.get_bit(23)) src1_data.mask_or(0xFFFFFFFF, 0xFF000000); - if (src2_data.get_bit(23)) src2_data.mask_or(0xFFFFFFFF, 0xFF000000); + switch ( i_type ) { + case S32_TYPE: + if( src1_data.get_bit(23) ) + src1_data.mask_or(0xFFFFFFFF,0xFF000000); + if( src2_data.get_bit(23) ) + src2_data.mask_or(0xFFFFFFFF,0xFF000000); data.s64 = src1_data.s64 * src2_data.s64; break; - case U32_TYPE: + case U32_TYPE: data.u64 = src1_data.u64 * src2_data.u64; break; - default: - printf( - "GPGPU-Sim PTX: Execution error - type mismatch with instruction\n"); + default: + printf("GPGPU-Sim PTX: Execution error - type mismatch with instruction\n"); assert(0); break; - } + } - if (pI->is_hi()) { - data.u64 = data.u64 >> 16; - data.mask_and(0, 0xFFFFFFFF); - } else if (pI->is_lo()) { - data.mask_and(0, 0xFFFFFFFF); - } + if ( pI->is_hi() ) { + data.u64 = data.u64 >> 16; + data.mask_and(0,0xFFFFFFFF); + } else if (pI->is_lo()) { + data.mask_and(0,0xFFFFFFFF); + } - thread->set_operand_value(dst, data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void mul_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t data; +void mul_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - ptx_reg_t d, t; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + ptx_reg_t d, t; - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - unsigned rounding_mode = pI->rounding_mode(); + unsigned rounding_mode = pI->rounding_mode(); - switch (i_type) { - case S16_TYPE: + switch ( i_type ) { + case S16_TYPE: t.s32 = ((int)a.s16) * ((int)b.s16); - if (pI->is_wide()) - d.s32 = t.s32; - else if (pI->is_hi()) - d.s16 = (t.s32 >> 16); - else if (pI->is_lo()) - d.s16 = t.s16; - else - assert(0); + if ( pI->is_wide() ) d.s32 = t.s32; + else if ( pI->is_hi() ) d.s16 = (t.s32>>16); + else if ( pI->is_lo() ) d.s16 = t.s16; + else assert(0); break; - case S32_TYPE: + case S32_TYPE: t.s64 = ((long long)a.s32) * ((long long)b.s32); - if (pI->is_wide()) - d.s64 = t.s64; - else if (pI->is_hi()) - d.s32 = (t.s64 >> 32); - else if (pI->is_lo()) - d.s32 = t.s32; - else - assert(0); + if ( pI->is_wide() ) d.s64 = t.s64; + else if ( pI->is_hi() ) d.s32 = (t.s64>>32); + else if ( pI->is_lo() ) d.s32 = t.s32; + else assert(0); break; - case S64_TYPE: + case S64_TYPE: t.s64 = a.s64 * b.s64; - assert(!pI->is_wide()); - assert(!pI->is_hi()); - if (pI->is_lo()) - d.s64 = t.s64; - else - assert(0); + assert( !pI->is_wide() ); + assert( !pI->is_hi() ); + if ( pI->is_lo() ) d.s64 = t.s64; + else assert(0); break; - case U16_TYPE: + case U16_TYPE: t.u32 = ((unsigned)a.u16) * ((unsigned)b.u16); - if (pI->is_wide()) - d.u32 = t.u32; - else if (pI->is_lo()) - d.u16 = t.u16; - else if (pI->is_hi()) - d.u16 = (t.u32 >> 16); - else - assert(0); + if ( pI->is_wide() ) d.u32 = t.u32; + else if ( pI->is_lo() ) d.u16 = t.u16; + else if ( pI->is_hi() ) d.u16 = (t.u32>>16); + else assert(0); break; - case U32_TYPE: + case U32_TYPE: t.u64 = ((unsigned long long)a.u32) * ((unsigned long long)b.u32); - if (pI->is_wide()) - d.u64 = t.u64; - else if (pI->is_lo()) - d.u32 = t.u32; - else if (pI->is_hi()) - d.u32 = (t.u64 >> 32); - else - assert(0); + if ( pI->is_wide() ) d.u64 = t.u64; + else if ( pI->is_lo() ) d.u32 = t.u32; + else if ( pI->is_hi() ) d.u32 = (t.u64>>32); + else assert(0); break; - case U64_TYPE: + case U64_TYPE: t.u64 = a.u64 * b.u64; - assert(!pI->is_wide()); - assert(!pI->is_hi()); - if (pI->is_lo()) - d.u64 = t.u64; - else - assert(0); + assert( !pI->is_wide() ); + assert( !pI->is_hi() ); + if ( pI->is_lo() ) d.u64 = t.u64; + else assert(0); break; - case F16_TYPE: { - // assert(0); - // break; - int orig_rm = fegetround(); - switch (rounding_mode) { - case RN_OPTION: - break; - case RZ_OPTION: - fesetround(FE_TOWARDZERO); - break; - default: - assert(0); - break; - } + case F16_TYPE:{ + //assert(0); + //break; + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } - d.f16 = a.f16 * b.f16; + d.f16 = a.f16 * b.f16; - if (pI->saturation_mode()) { - if (d.f16 < 0) - d.f16 = 0; - else if (d.f16 > 1.0f) - d.f16 = 1.0f; - } - fesetround(orig_rm); - break; - } - case F32_TYPE: { - int orig_rm = fegetround(); - switch (rounding_mode) { - case RN_OPTION: - break; - case RZ_OPTION: - fesetround(FE_TOWARDZERO); - break; - default: - assert(0); - break; - } + if ( pI->saturation_mode() ) { + if ( d.f16 < 0 ) d.f16 = 0; + else if ( d.f16 > 1.0f ) d.f16 = 1.0f; + } + fesetround( orig_rm ); + break; + } + case F32_TYPE: { + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } - d.f32 = a.f32 * b.f32; + d.f32 = a.f32 * b.f32; - if (pI->saturation_mode()) { - if (d.f32 < 0) - d.f32 = 0; - else if (d.f32 > 1.0f) - d.f32 = 1.0f; + if ( pI->saturation_mode() ) { + if ( d.f32 < 0 ) d.f32 = 0; + else if ( d.f32 > 1.0f ) d.f32 = 1.0f; + } + fesetround( orig_rm ); + break; + } + case F64_TYPE: case FF64_TYPE:{ + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + d.f64 = a.f64 * b.f64; + if ( pI->saturation_mode() ) { + if ( d.f64 < 0 ) d.f64 = 0; + else if ( d.f64 > 1.0f ) d.f64 = 1.0; + } + fesetround( orig_rm ); + break; } - fesetround(orig_rm); + default: + assert(0); break; - } - case F64_TYPE: - case FF64_TYPE: { - int orig_rm = fegetround(); - switch (rounding_mode) { - case RN_OPTION: - break; - case RZ_OPTION: - fesetround(FE_TOWARDZERO); - break; - default: - assert(0); - break; - } - d.f64 = a.f64 * b.f64; - if (pI->saturation_mode()) { - if (d.f64 < 0) - d.f64 = 0; - else if (d.f64 > 1.0f) - d.f64 = 1.0; - } - fesetround(orig_rm); - break; - } - default: - assert(0); - break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void neg_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; +void neg_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned to_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, to_type, thread, 1); + unsigned to_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, to_type, thread, 1); - switch (to_type) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - data.s64 = 0 - src1_data.s64; - break; // seems buggy, but not (just ignore higher bits) - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - assert(0); - break; - case F16_TYPE: - data.f16 = 0.0f - src1_data.f16; - break; // assert(0); break; - case F32_TYPE: - data.f32 = 0.0f - src1_data.f32; - break; - case F64_TYPE: - case FF64_TYPE: - data.f64 = 0.0f - src1_data.f64; - break; - default: - assert(0); - break; - } - thread->set_operand_value(dst, data, to_type, thread, pI); + switch ( to_type ) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + data.s64 = 0 - src1_data.s64; break; // seems buggy, but not (just ignore higher bits) + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + assert(0); break; + case F16_TYPE: data.f16 =0.0f - src1_data.f16; break;//assert(0); break; + case F32_TYPE: data.f32 = 0.0f - src1_data.f32; break; + case F64_TYPE: case FF64_TYPE: data.f64 = 0.0f - src1_data.f64; break; + default: assert(0); break; + } + + thread->set_operand_value(dst,data, to_type, thread, pI); } -// nandn bitwise negates second operand then bitwise nands with the first -// operand -void nandn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; +//nandn bitwise negates second operand then bitwise nands with the first operand +void nandn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - // the way ptxplus handles predicates: 1 = false and 0 = true - if (i_type == PRED_TYPE) - data.pred = (~src1_data.pred & src2_data.pred); - else - data.u64 = ~(src1_data.u64 & ~src2_data.u64); + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = (~src1_data.pred & src2_data.pred); + else + data.u64 = ~(src1_data.u64 & ~src2_data.u64); + + thread->set_operand_value(dst,data, i_type, thread, pI); - thread->set_operand_value(dst, data, i_type, thread, pI); } -// norn bitwise negates first operand then bitwise ands with the second operand -void norn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; +//norn bitwise negates first operand then bitwise ands with the second operand +void norn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - // the way ptxplus handles predicates: 1 = false and 0 = true - if (i_type == PRED_TYPE) - data.pred = ~(src1_data.pred & ~(src2_data.pred)); - else - data.u64 = ~(src1_data.u64) & src2_data.u64; + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = ~(src1_data.pred & ~(src2_data.pred)); + else + data.u64 = ~(src1_data.u64) & src2_data.u64; + + thread->set_operand_value(dst,data, i_type, thread, pI); - thread->set_operand_value(dst, data, i_type, thread, pI); } -void not_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void not_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch (i_type) { - case PRED_TYPE: - d.pred = (~(a.pred) & 0x000F); - break; - case B16_TYPE: - d.u16 = ~a.u16; - break; - case B32_TYPE: - d.u32 = ~a.u32; - break; - case B64_TYPE: - d.u64 = ~a.u64; - break; - default: + + switch ( i_type ) { + case PRED_TYPE: d.pred = (~(a.pred) & 0x000F); break; + case B16_TYPE: d.u16 = ~a.u16; break; + case B32_TYPE: d.u32 = ~a.u32; break; + case B64_TYPE: d.u64 = ~a.u64; break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -void or_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void or_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - // the way ptxplus handles predicates: 1 = false and 0 = true - if (i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) | ~(src2_data.pred)); - else - data.u64 = src1_data.u64 | src2_data.u64; + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) | ~(src2_data.pred)); + else + data.u64 = src1_data.u64 | src2_data.u64; - thread->set_operand_value(dst, data, i_type, thread, pI); + thread->set_operand_value(dst,data, i_type, thread, pI); } -void orn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void orn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - // the way ptxplus handles predicates: 1 = false and 0 = true - if (i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) | (src2_data.pred)); - else - data.u64 = src1_data.u64 | ~src2_data.u64; + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) | (src2_data.pred)); + else + data.u64 = src1_data.u64 | ~src2_data.u64; - thread->set_operand_value(dst, data, i_type, thread, pI); + thread->set_operand_value(dst,data, i_type, thread, pI); } -void pmevent_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void popc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src = pI->src1(); +void pmevent_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void popc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src = pI->src1(); - unsigned i_type = pI->get_type(); - src_data = thread->get_operand_value(src, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src_data = thread->get_operand_value(src, dst, i_type, thread, 1); - switch (i_type) { - case B32_TYPE: { - std::bitset<32> mask(src_data.u32); - data.u32 = mask.count(); - } break; - case B64_TYPE: { - std::bitset<64> mask(src_data.u64); + switch ( i_type ) { + case B32_TYPE: { + std::bitset<32> mask(src_data.u32); + data.u32 = mask.count(); + } break; + case B64_TYPE: { + std::bitset<64> mask(src_data.u64); data.u32 = mask.count(); - } break; - default: + } break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, data, i_type, thread, pI); -} -void prefetch_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void prefetchu_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); + thread->set_operand_value(dst,data, i_type, thread, pI); } +void prefetch_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void prefetchu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -int prmt_mode_present(int mode) { - int returnval = 0; - switch (mode) { - case PRMT_F4E_MODE: - case PRMT_B4E_MODE: - case PRMT_RC8_MODE: - case PRMT_RC16_MODE: - case PRMT_ECL_MODE: - case PRMT_ECR_MODE: - returnval = 1; - break; - default: - break; - } - return returnval; +int prmt_mode_present(int mode) +{ + int returnval=0; + switch(mode){ + case PRMT_F4E_MODE: + case PRMT_B4E_MODE: + case PRMT_RC8_MODE: + case PRMT_RC16_MODE: + case PRMT_ECL_MODE: + case PRMT_ECR_MODE: + returnval=1; + break; + default: + break; + } + return returnval; } -int read_byte(int mode, int control, int d_sel_index, signed long long value) { - int returnval = 0; - int prmt_f4e_mode[4][4] = { - {0, 1, 2, 3}, {1, 2, 3, 4}, {2, 3, 4, 5}, {3, 4, 5, 6}}; - int prmt_b4e_mode[4][4] = { - {0, 7, 6, 5}, {1, 0, 7, 6}, {2, 1, 0, 7}, {3, 2, 1, 0}}; - int prmt_rc8_mode[4][4] = { - {0, 0, 0, 0}, {1, 1, 1, 1}, {2, 2, 2, 2}, {3, 3, 3, 3}}; - int prmt_ecl_mode[4][4] = { - {0, 1, 2, 3}, {1, 1, 2, 3}, {2, 2, 2, 3}, {3, 3, 3, 3}}; - int prmt_ecr_mode[4][4] = { - {0, 0, 0, 0}, {0, 1, 1, 1}, {0, 1, 2, 2}, {0, 1, 2, 3}}; - int prmt_rc16_mode[4][4] = { - {0, 1, 0, 1}, {2, 3, 2, 3}, {0, 1, 0, 1}, {2, 3, 2, 3}}; +int read_byte(int mode, int control, int d_sel_index, signed long long value){ - if (!prmt_mode_present(mode)) { - if (control & 0x8) { - returnval = 0xff; - } else { - returnval = (value >> (8 * control)) & 0xff; - } - } else { - switch (mode) { - case PRMT_F4E_MODE: - returnval = prmt_f4e_mode[control][d_sel_index]; - break; - case PRMT_B4E_MODE: - returnval = prmt_b4e_mode[control][d_sel_index]; - break; - case PRMT_RC8_MODE: - returnval = prmt_rc8_mode[control][d_sel_index]; - break; - case PRMT_ECL_MODE: - returnval = prmt_ecl_mode[control][d_sel_index]; - break; - case PRMT_ECR_MODE: - returnval = prmt_ecr_mode[control][d_sel_index]; - break; - case PRMT_RC16_MODE: - returnval = prmt_rc16_mode[control][d_sel_index]; - break; - // Change the default from printing "ERROR" to just asserting - default: - assert(false); - } - } - return (returnval << 8 * d_sel_index); + int returnval = 0; + int prmt_f4e_mode[4][4]={{0,1,2,3},{1,2,3,4},{2,3,4,5},{3,4,5,6}}; + int prmt_b4e_mode[4][4]={{0,7,6,5},{1,0,7,6},{2,1,0,7},{3,2,1,0}}; + int prmt_rc8_mode[4][4]={{0,0,0,0},{1,1,1,1},{2,2,2,2},{3,3,3,3}}; + int prmt_ecl_mode[4][4]={{0,1,2,3},{1,1,2,3},{2,2,2,3},{3,3,3,3}}; + int prmt_ecr_mode[4][4]={{0,0,0,0},{0,1,1,1},{0,1,2,2},{0,1,2,3}}; + int prmt_rc16_mode[4][4]={{0,1,0,1},{2,3,2,3},{0,1,0,1},{2,3,2,3}}; + + if(!prmt_mode_present(mode)){ + if(control&0x8){ + returnval=0xff; + } + else{ + returnval= (value>>(8*control)) & 0xff; + } + } + else{ + switch(mode){ + case PRMT_F4E_MODE: returnval=prmt_f4e_mode[control][d_sel_index];break; + case PRMT_B4E_MODE: returnval=prmt_b4e_mode[control][d_sel_index];break; + case PRMT_RC8_MODE: returnval=prmt_rc8_mode[control][d_sel_index];break; + case PRMT_ECL_MODE: returnval=prmt_ecl_mode[control][d_sel_index];break; + case PRMT_ECR_MODE: returnval=prmt_ecr_mode[control][d_sel_index];break; + case PRMT_RC16_MODE: returnval=prmt_rc16_mode[control][d_sel_index];break; + // Change the default from printing "ERROR" to just asserting + default: assert(false); + } + } + return (returnval << 8 * d_sel_index); } -void prmt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, src3_data, tmpdata, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); +void prmt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { + + ptx_reg_t src1_data, src2_data, src3_data,tmpdata,data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); - unsigned mode = pI->prmt_op(); - unsigned i_type = pI->get_type(); + unsigned mode = pI->prmt_op(); + unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - tmpdata.s64 = src1_data.s32 | (src2_data.s64 << 32); - int ctl[4]; + tmpdata.s64=src1_data.s32|(src2_data.s64<<32); + int ctl[4]; - if (!prmt_mode_present(mode)) { - ctl[0] = (src3_data.s32 >> 0) & 0xf; - ctl[1] = (src3_data.s32 >> 4) & 0xf; - ctl[2] = (src3_data.s32 >> 8) & 0xf; - ctl[3] = (src3_data.s32 >> 12) & 0xf; - } else { - ctl[0] = ctl[1] = ctl[2] = ctl[3] = (src3_data.s32 >> 0) & 0x3; - } + if(!prmt_mode_present(mode)){ + ctl[0]=(src3_data.s32>>0)&0xf; + ctl[1]=(src3_data.s32>>4)&0xf; + ctl[2]=(src3_data.s32>>8)&0xf; + ctl[3]=(src3_data.s32>>12)&0xf; + } + else{ + ctl[0]=ctl[1]=ctl[2]=ctl[3]=(src3_data.s32>>0)&0x3; + } + + data.s32=0; + data.s32=data.s32|read_byte(mode,ctl[0],0,tmpdata.s64); //First byte-0 + data.s32=data.s32|read_byte(mode,ctl[1],1,tmpdata.s64); //Second byte-1 + data.s32=data.s32|read_byte(mode,ctl[2],2,tmpdata.s64); //Third byte-2 + data.s32=data.s32|read_byte(mode,ctl[3],3,tmpdata.s64); //Fourth byte-3 + + thread->set_operand_value(dst,data, i_type, thread, pI); - data.s32 = 0; - data.s32 = data.s32 | read_byte(mode, ctl[0], 0, tmpdata.s64); // First - // byte-0 - data.s32 = - data.s32 | read_byte(mode, ctl[1], 1, tmpdata.s64); // Second byte-1 - data.s32 = data.s32 | read_byte(mode, ctl[2], 2, tmpdata.s64); // Third - // byte-2 - data.s32 = - data.s32 | read_byte(mode, ctl[3], 3, tmpdata.s64); // Fourth byte-3 - thread->set_operand_value(dst, data, i_type, thread, pI); } -void rcp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void rcp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch (i_type) { - case F32_TYPE: + switch ( i_type ) { + case F32_TYPE: data.f32 = 1.0f / src1_data.f32; break; - case F64_TYPE: - case FF64_TYPE: + case F64_TYPE: + case FF64_TYPE: data.f64 = 1.0f / src1_data.f64; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, data, i_type, thread, pI); + thread->set_operand_value(dst,data, i_type, thread, pI); } -void red_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} +void red_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void rem_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; +void rem_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch (i_type) { - case S32_TYPE: + switch ( i_type ) { + case S32_TYPE: data.s32 = src1_data.s32 % src2_data.s32; break; - case S64_TYPE: + case S64_TYPE: data.s64 = src1_data.s64 % src2_data.s64; break; - case U32_TYPE: + case U32_TYPE: data.u32 = src1_data.u32 % src2_data.u32; break; - case U64_TYPE: + case U64_TYPE: data.u64 = src1_data.u64 % src2_data.u64; break; - default: - assert(0); - break; - } + default: assert(0); break; + } - thread->set_operand_value(dst, data, i_type, thread, pI); + thread->set_operand_value(dst,data, i_type, thread, pI); } -void ret_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - bool empty = thread->callstack_pop(); - if (empty) { - thread->set_done(); - thread->exitCore(); - thread->registerExit(); - } +void ret_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + bool empty = thread->callstack_pop(); + if( empty ) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); + } } -// Ptxplus version of ret instruction. -void retp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - bool empty = thread->callstack_pop_plus(); - if (empty) { - thread->set_done(); - thread->exitCore(); - thread->registerExit(); - } +//Ptxplus version of ret instruction. +void retp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + bool empty = thread->callstack_pop_plus(); + if( empty ) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); + } } -void rsqrt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void rsqrt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch (i_type) { - case F32_TYPE: - if (a.f32 < 0) { - d.u64 = 0; - d.u64 = 0x7fc00000; // NaN - } else if (a.f32 == 0) { - d.u64 = 0; - d.u32 = 0x7f800000; // Inf + switch ( i_type ) { + case F32_TYPE: + if ( a.f32 < 0 ) { + d.u64 = 0; + d.u64 = 0x7fc00000; // NaN + } else if ( a.f32 == 0 ) { + d.u64 = 0; + d.u32 = 0x7f800000; // Inf } else - d.f32 = cuda_math::__internal_accurate_fdividef(1.0f, sqrtf(a.f32)); + d.f32 = cuda_math::__internal_accurate_fdividef(1.0f, sqrtf(a.f32)); break; - case F64_TYPE: - case FF64_TYPE: - if (a.f32 < 0) { - d.u64 = 0; - d.u32 = 0x7fc00000; // NaN - float x = d.f32; - d.f64 = (double)x; - } else if (a.f32 == 0) { - d.u64 = 0; - d.u32 = 0x7f800000; // Inf - float x = d.f32; - d.f64 = (double)x; + case F64_TYPE: + case FF64_TYPE: + if ( a.f32 < 0 ) { + d.u64 = 0; + d.u32 = 0x7fc00000; // NaN + float x = d.f32; + d.f64 = (double)x; + } else if ( a.f32 == 0 ) { + d.u64 = 0; + d.u32 = 0x7f800000; // Inf + float x = d.f32; + d.f64 = (double)x; } else - d.f64 = 1.0 / sqrt(a.f64); + d.f64 = 1.0 / sqrt(a.f64); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -#define SAD(d, a, b, c) d = c + ((a < b) ? (b - a) : (a - b)) +#define SAD(d,a,b,c) d = c + ((a<b) ? (b-a) : (a-b)) -void sad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, b, c, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); +void sad_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, c, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, i_type, thread, 1); - switch (i_type) { - case U16_TYPE: - SAD(d.u16, a.u16, b.u16, c.u16); - break; - case U32_TYPE: - SAD(d.u32, a.u32, b.u32, c.u32); - break; - case U64_TYPE: - SAD(d.u64, a.u64, b.u64, c.u64); - break; - case S16_TYPE: - SAD(d.s16, a.s16, b.s16, c.s16); - break; - case S32_TYPE: - SAD(d.s32, a.s32, b.s32, c.s32); - break; - case S64_TYPE: - SAD(d.s64, a.s64, b.s64, c.s64); - break; - case F32_TYPE: - SAD(d.f32, a.f32, b.f32, c.f32); - break; - case F64_TYPE: - case FF64_TYPE: - SAD(d.f64, a.f64, b.f64, c.f64); - break; - default: + + switch ( i_type ) { + case U16_TYPE: SAD(d.u16,a.u16,b.u16,c.u16); break; + case U32_TYPE: SAD(d.u32,a.u32,b.u32,c.u32); break; + case U64_TYPE: SAD(d.u64,a.u64,b.u64,c.u64); break; + case S16_TYPE: SAD(d.s16,a.s16,b.s16,c.s16); break; + case S32_TYPE: SAD(d.s32,a.s32,b.s32,c.s32); break; + case S64_TYPE: SAD(d.s64,a.s64,b.s64,c.s64); break; + case F32_TYPE: SAD(d.f32,a.f32,b.f32,c.f32); break; + case F64_TYPE: case FF64_TYPE: SAD(d.f64,a.f64,b.f64,c.f64); break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -void selp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); +void selp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); - ptx_reg_t a, b, c, d; + ptx_reg_t a, b, c, d; - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, i_type, thread, 1); - // predicate value was changed so the lowest bit being set means the zero flag - // is set. - // As a result, the value of c.pred must be inverted to get proper behavior - d = (!(c.pred & 0x0001)) ? a : b; + //predicate value was changed so the lowest bit being set means the zero flag is set. + //As a result, the value of c.pred must be inverted to get proper behavior + d = (!(c.pred & 0x0001))?a:b; - thread->set_operand_value(dst, d, PRED_TYPE, thread, pI); + thread->set_operand_value(dst,d, PRED_TYPE, thread, pI); } -bool isFloat(int type) { - switch (type) { - case F16_TYPE: - case F32_TYPE: - case F64_TYPE: - case FF64_TYPE: +bool isFloat(int type) +{ + switch ( type ) { + case F16_TYPE: + case F32_TYPE: + case F64_TYPE: + case FF64_TYPE: return true; - default: + default: return false; - } + } } -bool CmpOp(int type, ptx_reg_t a, ptx_reg_t b, unsigned cmpop) { - bool t = false; +bool CmpOp( int type, ptx_reg_t a, ptx_reg_t b, unsigned cmpop ) +{ + bool t = false; - switch (type) { - case B16_TYPE: + switch ( type ) { + case B16_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.u16 == b.u16); - break; - case NE_OPTION: - t = (a.u16 != b.u16); - break; - default: - assert(0); + case EQ_OPTION: t = (a.u16 == b.u16); break; + case NE_OPTION: t = (a.u16 != b.u16); break; + default: + assert(0); } - case B32_TYPE: + case B32_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.u32 == b.u32); - break; - case NE_OPTION: - t = (a.u32 != b.u32); - break; - default: - assert(0); + case EQ_OPTION: t = (a.u32 == b.u32); break; + case NE_OPTION: t = (a.u32 != b.u32); break; + default: + assert(0); } - case B64_TYPE: + case B64_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.u64 == b.u64); - break; - case NE_OPTION: - t = (a.u64 != b.u64); - break; - default: - assert(0); + case EQ_OPTION: t = (a.u64 == b.u64); break; + case NE_OPTION: t = (a.u64 != b.u64); break; + default: + assert(0); } break; - case S8_TYPE: - case S16_TYPE: + case S8_TYPE: + case S16_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.s16 == b.s16); - break; - case NE_OPTION: - t = (a.s16 != b.s16); - break; - case LT_OPTION: - t = (a.s16 < b.s16); - break; - case LE_OPTION: - t = (a.s16 <= b.s16); - break; - case GT_OPTION: - t = (a.s16 > b.s16); - break; - case GE_OPTION: - t = (a.s16 >= b.s16); - break; - default: - assert(0); + case EQ_OPTION: t = (a.s16 == b.s16); break; + case NE_OPTION: t = (a.s16 != b.s16); break; + case LT_OPTION: t = (a.s16 < b.s16); break; + case LE_OPTION: t = (a.s16 <= b.s16); break; + case GT_OPTION: t = (a.s16 > b.s16); break; + case GE_OPTION: t = (a.s16 >= b.s16); break; + default: + assert(0); } break; - case S32_TYPE: + case S32_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.s32 == b.s32); - break; - case NE_OPTION: - t = (a.s32 != b.s32); - break; - case LT_OPTION: - t = (a.s32 < b.s32); - break; - case LE_OPTION: - t = (a.s32 <= b.s32); - break; - case GT_OPTION: - t = (a.s32 > b.s32); - break; - case GE_OPTION: - t = (a.s32 >= b.s32); - break; - default: - assert(0); + case EQ_OPTION: t = (a.s32 == b.s32); break; + case NE_OPTION: t = (a.s32 != b.s32); break; + case LT_OPTION: t = (a.s32 < b.s32); break; + case LE_OPTION: t = (a.s32 <= b.s32); break; + case GT_OPTION: t = (a.s32 > b.s32); break; + case GE_OPTION: t = (a.s32 >= b.s32); break; + default: + assert(0); } break; - case S64_TYPE: + case S64_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.s64 == b.s64); - break; - case NE_OPTION: - t = (a.s64 != b.s64); - break; - case LT_OPTION: - t = (a.s64 < b.s64); - break; - case LE_OPTION: - t = (a.s64 <= b.s64); - break; - case GT_OPTION: - t = (a.s64 > b.s64); - break; - case GE_OPTION: - t = (a.s64 >= b.s64); - break; - default: - assert(0); + case EQ_OPTION: t = (a.s64 == b.s64); break; + case NE_OPTION: t = (a.s64 != b.s64); break; + case LT_OPTION: t = (a.s64 < b.s64); break; + case LE_OPTION: t = (a.s64 <= b.s64); break; + case GT_OPTION: t = (a.s64 > b.s64); break; + case GE_OPTION: t = (a.s64 >= b.s64); break; + default: + assert(0); } break; - case U8_TYPE: - case U16_TYPE: + case U8_TYPE: + case U16_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.u16 == b.u16); - break; - case NE_OPTION: - t = (a.u16 != b.u16); - break; - case LT_OPTION: - t = (a.u16 < b.u16); - break; - case LE_OPTION: - t = (a.u16 <= b.u16); - break; - case GT_OPTION: - t = (a.u16 > b.u16); - break; - case GE_OPTION: - t = (a.u16 >= b.u16); - break; - case LO_OPTION: - t = (a.u16 < b.u16); - break; - case LS_OPTION: - t = (a.u16 <= b.u16); - break; - case HI_OPTION: - t = (a.u16 > b.u16); - break; - case HS_OPTION: - t = (a.u16 >= b.u16); - break; - default: - assert(0); + case EQ_OPTION: t = (a.u16 == b.u16); break; + case NE_OPTION: t = (a.u16 != b.u16); break; + case LT_OPTION: t = (a.u16 < b.u16); break; + case LE_OPTION: t = (a.u16 <= b.u16); break; + case GT_OPTION: t = (a.u16 > b.u16); break; + case GE_OPTION: t = (a.u16 >= b.u16); break; + case LO_OPTION: t = (a.u16 < b.u16); break; + case LS_OPTION: t = (a.u16 <= b.u16); break; + case HI_OPTION: t = (a.u16 > b.u16); break; + case HS_OPTION: t = (a.u16 >= b.u16); break; + default: + assert(0); } break; - case U32_TYPE: + case U32_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.u32 == b.u32); - break; - case NE_OPTION: - t = (a.u32 != b.u32); - break; - case LT_OPTION: - t = (a.u32 < b.u32); - break; - case LE_OPTION: - t = (a.u32 <= b.u32); - break; - case GT_OPTION: - t = (a.u32 > b.u32); - break; - case GE_OPTION: - t = (a.u32 >= b.u32); - break; - case LO_OPTION: - t = (a.u32 < b.u32); - break; - case LS_OPTION: - t = (a.u32 <= b.u32); - break; - case HI_OPTION: - t = (a.u32 > b.u32); - break; - case HS_OPTION: - t = (a.u32 >= b.u32); - break; - default: - assert(0); + case EQ_OPTION: t = (a.u32 == b.u32); break; + case NE_OPTION: t = (a.u32 != b.u32); break; + case LT_OPTION: t = (a.u32 < b.u32); break; + case LE_OPTION: t = (a.u32 <= b.u32); break; + case GT_OPTION: t = (a.u32 > b.u32); break; + case GE_OPTION: t = (a.u32 >= b.u32); break; + case LO_OPTION: t = (a.u32 < b.u32); break; + case LS_OPTION: t = (a.u32 <= b.u32); break; + case HI_OPTION: t = (a.u32 > b.u32); break; + case HS_OPTION: t = (a.u32 >= b.u32); break; + default: + assert(0); } break; - case U64_TYPE: + case U64_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.u64 == b.u64); - break; - case NE_OPTION: - t = (a.u64 != b.u64); - break; - case LT_OPTION: - t = (a.u64 < b.u64); - break; - case LE_OPTION: - t = (a.u64 <= b.u64); - break; - case GT_OPTION: - t = (a.u64 > b.u64); - break; - case GE_OPTION: - t = (a.u64 >= b.u64); - break; - case LO_OPTION: - t = (a.u64 < b.u64); - break; - case LS_OPTION: - t = (a.u64 <= b.u64); - break; - case HI_OPTION: - t = (a.u64 > b.u64); - break; - case HS_OPTION: - t = (a.u64 >= b.u64); - break; - default: - assert(0); + case EQ_OPTION: t = (a.u64 == b.u64); break; + case NE_OPTION: t = (a.u64 != b.u64); break; + case LT_OPTION: t = (a.u64 < b.u64); break; + case LE_OPTION: t = (a.u64 <= b.u64); break; + case GT_OPTION: t = (a.u64 > b.u64); break; + case GE_OPTION: t = (a.u64 >= b.u64); break; + case LO_OPTION: t = (a.u64 < b.u64); break; + case LS_OPTION: t = (a.u64 <= b.u64); break; + case HI_OPTION: t = (a.u64 > b.u64); break; + case HS_OPTION: t = (a.u64 >= b.u64); break; + default: + assert(0); } break; - case F16_TYPE: - assert(0); - break; - case F32_TYPE: + case F16_TYPE: assert(0); break; + case F32_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.f32 == b.f32) && !isNaN(a.f32) && !isNaN(b.f32); - break; - case NE_OPTION: - t = (a.f32 != b.f32) && !isNaN(a.f32) && !isNaN(b.f32); - break; - case LT_OPTION: - t = (a.f32 < b.f32) && !isNaN(a.f32) && !isNaN(b.f32); - break; - case LE_OPTION: - t = (a.f32 <= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); - break; - case GT_OPTION: - t = (a.f32 > b.f32) && !isNaN(a.f32) && !isNaN(b.f32); - break; - case GE_OPTION: - t = (a.f32 >= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); - break; - case EQU_OPTION: - t = (a.f32 == b.f32) || isNaN(a.f32) || isNaN(b.f32); - break; - case NEU_OPTION: - t = (a.f32 != b.f32) || isNaN(a.f32) || isNaN(b.f32); - break; - case LTU_OPTION: - t = (a.f32 < b.f32) || isNaN(a.f32) || isNaN(b.f32); - break; - case LEU_OPTION: - t = (a.f32 <= b.f32) || isNaN(a.f32) || isNaN(b.f32); - break; - case GTU_OPTION: - t = (a.f32 > b.f32) || isNaN(a.f32) || isNaN(b.f32); - break; - case GEU_OPTION: - t = (a.f32 >= b.f32) || isNaN(a.f32) || isNaN(b.f32); - break; - case NUM_OPTION: - t = !isNaN(a.f32) && !isNaN(b.f32); - break; - case NAN_OPTION: - t = isNaN(a.f32) || isNaN(b.f32); - break; - default: - assert(0); + case EQ_OPTION: t = (a.f32 == b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; + case NE_OPTION: t = (a.f32 != b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; + case LT_OPTION: t = (a.f32 < b.f32 ) && !isNaN(a.f32) && !isNaN(b.f32); break; + case LE_OPTION: t = (a.f32 <= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; + case GT_OPTION: t = (a.f32 > b.f32 ) && !isNaN(a.f32) && !isNaN(b.f32); break; + case GE_OPTION: t = (a.f32 >= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; + case EQU_OPTION: t = (a.f32 == b.f32) || isNaN(a.f32) || isNaN(b.f32); break; + case NEU_OPTION: t = (a.f32 != b.f32) || isNaN(a.f32) || isNaN(b.f32); break; + case LTU_OPTION: t = (a.f32 < b.f32 ) || isNaN(a.f32) || isNaN(b.f32); break; + case LEU_OPTION: t = (a.f32 <= b.f32) || isNaN(a.f32) || isNaN(b.f32); break; + case GTU_OPTION: t = (a.f32 > b.f32 ) || isNaN(a.f32) || isNaN(b.f32); break; + case GEU_OPTION: t = (a.f32 >= b.f32) || isNaN(a.f32) || isNaN(b.f32); break; + case NUM_OPTION: t = !isNaN(a.f32) && !isNaN(b.f32); break; + case NAN_OPTION: t = isNaN(a.f32) || isNaN(b.f32); break; + default: + assert(0); } break; - case F64_TYPE: - case FF64_TYPE: + case F64_TYPE: + case FF64_TYPE: switch (cmpop) { - case EQ_OPTION: - t = (a.f64 == b.f64) && !isNaN(a.f64) && !isNaN(b.f64); - break; - case NE_OPTION: - t = (a.f64 != b.f64) && !isNaN(a.f64) && !isNaN(b.f64); - break; - case LT_OPTION: - t = (a.f64 < b.f64) && !isNaN(a.f64) && !isNaN(b.f64); - break; - case LE_OPTION: - t = (a.f64 <= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); - break; - case GT_OPTION: - t = (a.f64 > b.f64) && !isNaN(a.f64) && !isNaN(b.f64); - break; - case GE_OPTION: - t = (a.f64 >= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); - break; - case EQU_OPTION: - t = (a.f64 == b.f64) || isNaN(a.f64) || isNaN(b.f64); - break; - case NEU_OPTION: - t = (a.f64 != b.f64) || isNaN(a.f64) || isNaN(b.f64); - break; - case LTU_OPTION: - t = (a.f64 < b.f64) || isNaN(a.f64) || isNaN(b.f64); - break; - case LEU_OPTION: - t = (a.f64 <= b.f64) || isNaN(a.f64) || isNaN(b.f64); - break; - case GTU_OPTION: - t = (a.f64 > b.f64) || isNaN(a.f64) || isNaN(b.f64); - break; - case GEU_OPTION: - t = (a.f64 >= b.f64) || isNaN(a.f64) || isNaN(b.f64); - break; - case NUM_OPTION: - t = !isNaN(a.f64) && !isNaN(b.f64); - break; - case NAN_OPTION: - t = isNaN(a.f64) || isNaN(b.f64); - break; - default: - assert(0); + case EQ_OPTION: t = (a.f64 == b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; + case NE_OPTION: t = (a.f64 != b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; + case LT_OPTION: t = (a.f64 < b.f64 ) && !isNaN(a.f64) && !isNaN(b.f64); break; + case LE_OPTION: t = (a.f64 <= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; + case GT_OPTION: t = (a.f64 > b.f64 ) && !isNaN(a.f64) && !isNaN(b.f64); break; + case GE_OPTION: t = (a.f64 >= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; + case EQU_OPTION: t = (a.f64 == b.f64) || isNaN(a.f64) || isNaN(b.f64); break; + case NEU_OPTION: t = (a.f64 != b.f64) || isNaN(a.f64) || isNaN(b.f64); break; + case LTU_OPTION: t = (a.f64 < b.f64 ) || isNaN(a.f64) || isNaN(b.f64); break; + case LEU_OPTION: t = (a.f64 <= b.f64) || isNaN(a.f64) || isNaN(b.f64); break; + case GTU_OPTION: t = (a.f64 > b.f64 ) || isNaN(a.f64) || isNaN(b.f64); break; + case GEU_OPTION: t = (a.f64 >= b.f64) || isNaN(a.f64) || isNaN(b.f64); break; + case NUM_OPTION: t = !isNaN(a.f64) && !isNaN(b.f64); break; + case NAN_OPTION: t = isNaN(a.f64) || isNaN(b.f64); break; + default: + assert(0); } break; - default: - assert(0); - break; - } + default: assert(0); break; + } - return t; + return t; } -void setp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, b; +void setp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b; - int t = 0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + int t=0; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - assert(pI->get_num_operands() < - 4); // or need to deal with "c" operand / boolOp + assert( pI->get_num_operands() < 4 ); // or need to deal with "c" operand / boolOp - unsigned type = pI->get_type(); - unsigned cmpop = pI->get_cmpop(); - a = thread->get_operand_value(src1, dst, type, thread, 1); - b = thread->get_operand_value(src2, dst, type, thread, 1); + unsigned type = pI->get_type(); + unsigned cmpop = pI->get_cmpop(); + a = thread->get_operand_value(src1, dst, type, thread, 1); + b = thread->get_operand_value(src2, dst, type, thread, 1); - t = CmpOp(type, a, b, cmpop); + t = CmpOp(type,a,b,cmpop); - ptx_reg_t data; + ptx_reg_t data; - // the way ptxplus handles the zero flag, 1 = false and 0 = true - data.pred = - (t == - 0); // inverting predicate since ptxplus uses "1" for a set zero flag + //the way ptxplus handles the zero flag, 1 = false and 0 = true + data.pred = (t==0); //inverting predicate since ptxplus uses "1" for a set zero flag - thread->set_operand_value(dst, data, PRED_TYPE, thread, pI); + thread->set_operand_value(dst,data, PRED_TYPE, thread, pI); } -void set_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, b; +void set_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b; - int t = 0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + int t=0; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - assert(pI->get_num_operands() < - 4); // or need to deal with "c" operand / boolOp + assert( pI->get_num_operands() < 4 ); // or need to deal with "c" operand / boolOp - unsigned src_type = pI->get_type2(); - unsigned cmpop = pI->get_cmpop(); + unsigned src_type = pI->get_type2(); + unsigned cmpop = pI->get_cmpop(); - a = thread->get_operand_value(src1, dst, src_type, thread, 1); - b = thread->get_operand_value(src2, dst, src_type, thread, 1); + a = thread->get_operand_value(src1, dst, src_type, thread, 1); + b = thread->get_operand_value(src2, dst, src_type, thread, 1); - // Take abs of first operand if needed - if (pI->is_abs()) { - switch (src_type) { - case S16_TYPE: - a.s16 = my_abs(a.s16); - break; - case S32_TYPE: - a.s32 = my_abs(a.s32); - break; - case S64_TYPE: - a.s64 = my_abs(a.s64); - break; - case U16_TYPE: - a.u16 = a.u16; - break; - case U32_TYPE: - a.u32 = my_abs(a.u32); - break; - case U64_TYPE: - a.u64 = my_abs(a.u64); - break; - case F32_TYPE: - a.f32 = my_abs(a.f32); - break; - case F64_TYPE: - case FF64_TYPE: - a.f64 = my_abs(a.f64); - break; + // Take abs of first operand if needed + if(pI->is_abs()) { + switch ( src_type ) { + case S16_TYPE: a.s16 = my_abs(a.s16); break; + case S32_TYPE: a.s32 = my_abs(a.s32); break; + case S64_TYPE: a.s64 = my_abs(a.s64); break; + case U16_TYPE: a.u16 = a.u16; break; + case U32_TYPE: a.u32 = my_abs(a.u32); break; + case U64_TYPE: a.u64 = my_abs(a.u64); break; + case F32_TYPE: a.f32 = my_abs(a.f32); break; + case F64_TYPE: case FF64_TYPE: a.f64 = my_abs(a.f64); break; default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - } + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + } - t = CmpOp(src_type, a, b, cmpop); + t = CmpOp(src_type,a,b,cmpop); - ptx_reg_t data; - if (isFloat(pI->get_type())) { - data.f32 = (t != 0) ? 1.0f : 0.0f; - } else { - data.u32 = (t != 0) ? 0xFFFFFFFF : 0; - } + ptx_reg_t data; + if ( isFloat(pI->get_type()) ) { + data.f32 = (t!=0)?1.0f:0.0f; + } else { + data.u32 = (t!=0)?0xFFFFFFFF:0; + } - thread->set_operand_value(dst, data, pI->get_type(), thread, pI); -} + thread->set_operand_value(dst, data, pI->get_type(), thread, pI); -void shfl_impl(const ptx_instruction *pI, core_t *core, warp_inst_t inst) { - unsigned i_type = pI->get_type(); - int tid; +} - if (core->get_gpu()->is_functional_sim()) - tid = inst.warp_id_func() * core->get_warp_size(); - else - tid = inst.warp_id() * core->get_warp_size(); +void shfl_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) +{ + unsigned i_type = pI->get_type(); + int tid; - ptx_thread_info *thread = core->get_thread_info()[tid]; - ptx_warp_info *warp_info = thread->m_warp_info; - int lane = warp_info->get_done_threads(); - thread = core->get_thread_info()[tid + lane]; + if(core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); + + ptx_thread_info *thread = core->get_thread_info()[tid]; + ptx_warp_info *warp_info = thread->m_warp_info; + int lane = warp_info->get_done_threads(); + thread = core->get_thread_info()[tid + lane]; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - int bval = (thread->get_operand_value(src2, dst, i_type, thread, 1)).u32; - int cval = (thread->get_operand_value(src3, dst, i_type, thread, 1)).u32; - int mask = cval >> 8; - bval &= 0x1F; - cval &= 0x1F; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + int bval = (thread->get_operand_value(src2, dst, i_type, thread, 1)).u32; + int cval = (thread->get_operand_value(src3, dst, i_type, thread, 1)).u32; + int mask = cval >> 8; + bval &= 0x1F; + cval &= 0x1F; - int maxLane = (lane & mask) | (cval & ~mask); - int minLane = lane & mask; + int maxLane = (lane & mask) | (cval & ~mask); + int minLane = lane & mask; - int src_idx; - unsigned p; - switch (pI->shfl_op()) { - case UP_OPTION: - src_idx = lane - bval; - p = (src_idx >= maxLane); - break; - case DOWN_OPTION: - src_idx = lane + bval; - p = (src_idx <= maxLane); - break; - case BFLY_OPTION: - src_idx = lane ^ bval; - p = (src_idx <= maxLane); - break; - case IDX_OPTION: - src_idx = minLane | (bval & ~mask); - p = (src_idx <= maxLane); - break; - default: - printf("GPGPU-Sim PTX: ERROR: Invalid shfl option\n"); - assert(0); - break; - } - // copy from own lane - if (!p) src_idx = lane; + int src_idx; + unsigned p; + switch(pI->shfl_op()) { + case UP_OPTION: + src_idx = lane - bval; + p = (src_idx >= maxLane); + break; + case DOWN_OPTION: + src_idx = lane + bval; + p = (src_idx <= maxLane); + break; + case BFLY_OPTION: + src_idx = lane ^ bval; + p = (src_idx <= maxLane); + break; + case IDX_OPTION: + src_idx = minLane | (bval & ~mask); + p = (src_idx <= maxLane); + break; + default: + printf("GPGPU-Sim PTX: ERROR: Invalid shfl option\n"); + assert(0); + break; + } + // copy from own lane + if (!p) src_idx = lane; - // copy input from lane src_idx - ptx_reg_t data; - if (inst.active(src_idx)) { - ptx_thread_info *source = core->get_thread_info()[tid + src_idx]; - data = source->get_operand_value(src1, dst, i_type, source, 1); - } else { - printf( - "GPGPU-Sim PTX: WARNING: shfl input value unpredictable for inactive " - "threads in a warp\n"); - data.u32 = 0; - } - thread->set_operand_value(dst, data, i_type, thread, pI); + // copy input from lane src_idx + ptx_reg_t data; + if (inst.active(src_idx)) { + ptx_thread_info *source = core->get_thread_info()[tid + src_idx]; + data = source->get_operand_value(src1, dst, i_type, source, 1); + } else { + printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for inactive threads in a warp\n"); + data.u32 = 0; + } + thread->set_operand_value(dst, data, i_type, thread, pI); - /* - TODO: deal with predicates appropriately using the following pseudocode: - if (!isGuardPredicateTrue(src_idx)) { - printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for - predicated-off threads in a warp\n"); - } - if (dest predicate selected) data.pred = p; - */ + /* + TODO: deal with predicates appropriately using the following pseudocode: + if (!isGuardPredicateTrue(src_idx)) { + printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for predicated-off threads in a warp\n"); + } + if (dest predicate selected) data.pred = p; + */ - // keep track of the number of threads that have executed in the warp - warp_info->inc_done_threads(); - if (warp_info->get_done_threads() == inst.active_count()) { - warp_info->reset_done_threads(); - } + // keep track of the number of threads that have executed in the warp + warp_info->inc_done_threads(); + if (warp_info->get_done_threads() == inst.active_count()) { + warp_info->reset_done_threads(); + } } -void shl_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void shl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch (i_type) { - case B16_TYPE: - case U16_TYPE: - if (b.u16 >= 16) - d.u16 = 0; + switch ( i_type ) { + case B16_TYPE: + case U16_TYPE: + if ( b.u16 >= 16 ) + d.u16 = 0; else - d.u16 = (unsigned short)((a.u16 << b.u16) & 0xFFFF); + d.u16 = (unsigned short) ((a.u16 << b.u16) & 0xFFFF); break; - case B32_TYPE: - case U32_TYPE: - if (b.u32 >= 32) - d.u32 = 0; + case B32_TYPE: + case U32_TYPE: + if ( b.u32 >= 32 ) + d.u32 = 0; else - d.u32 = (unsigned)((a.u32 << b.u32) & 0xFFFFFFFF); + d.u32 = (unsigned) ((a.u32 << b.u32) & 0xFFFFFFFF); break; - case B64_TYPE: - case U64_TYPE: - if (b.u32 >= 64) - d.u64 = 0; + case B64_TYPE: + case U64_TYPE: + if ( b.u32 >= 64 ) + d.u64 = 0; else - d.u64 = (a.u64 << b.u64); + d.u64 = (a.u64 << b.u64); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void shr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void shr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch (i_type) { - case U16_TYPE: - case B16_TYPE: - if (b.u16 < 16) - d.u16 = (unsigned short)((a.u16 >> b.u16) & 0xFFFF); + switch ( i_type ) { + case U16_TYPE: + case B16_TYPE: + if ( b.u16 < 16 ) + d.u16 = (unsigned short) ((a.u16 >> b.u16) & 0xFFFF); else - d.u16 = 0; + d.u16 = 0; break; - case U32_TYPE: - case B32_TYPE: - if (b.u32 < 32) - d.u32 = (unsigned)((a.u32 >> b.u32) & 0xFFFFFFFF); + case U32_TYPE: + case B32_TYPE: + if ( b.u32 < 32 ) + d.u32 = (unsigned) ((a.u32 >> b.u32) & 0xFFFFFFFF); else - d.u32 = 0; + d.u32 = 0; break; - case U64_TYPE: - case B64_TYPE: - if (b.u32 < 64) - d.u64 = (a.u64 >> b.u64); + case U64_TYPE: + case B64_TYPE: + if ( b.u32 < 64 ) + d.u64 = (a.u64 >> b.u64); else - d.u64 = 0; + d.u64 = 0; break; - case S16_TYPE: - if (b.u16 < 16) - d.s64 = (a.s16 >> b.s16); + case S16_TYPE: + if ( b.u16 < 16 ) + d.s64 = (a.s16 >> b.s16); else { - if (a.s16 < 0) { - d.s64 = -1; - } else { - d.s64 = 0; - } + if ( a.s16 < 0 ) { + d.s64 = -1; + } else { + d.s64 = 0; + } } break; - case S32_TYPE: - if (b.u32 < 32) - d.s64 = (a.s32 >> b.s32); + case S32_TYPE: + if ( b.u32 < 32 ) + d.s64 = (a.s32 >> b.s32); else { - if (a.s32 < 0) { - d.s64 = -1; - } else { - d.s64 = 0; - } + if ( a.s32 < 0 ) { + d.s64 = -1; + } else { + d.s64 = 0; + } } break; - case S64_TYPE: - if (b.u64 < 64) - d.s64 = (a.s64 >> b.u64); + case S64_TYPE: + if ( b.u64 < 64 ) + d.s64 = (a.s64 >> b.u64); else { - if (a.s64 < 0) { - if (b.s32 < 0) { - d.u64 = -1; - d.s32 = 0; - } else { - d.s64 = -1; - } - } else { - d.s64 = 0; - } + if ( a.s64 < 0 ) { + if ( b.s32 < 0 ) { + d.u64 = -1; + d.s32 = 0; + } else { + d.s64 = -1; + } + } else { + d.s64 = 0; + } } break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -void sin_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void sin_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch (i_type) { - case F32_TYPE: + switch ( i_type ) { + case F32_TYPE: d.f32 = sin(a.f32); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -void slct_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); +void slct_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); - ptx_reg_t a, b, c, d; + ptx_reg_t a, b, c, d; - unsigned i_type = pI->get_type(); - unsigned c_type = pI->get_type2(); - bool t = false; - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, c_type, thread, 1); + unsigned i_type = pI->get_type(); + unsigned c_type = pI->get_type2(); + bool t = false; + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, c_type, thread, 1); - switch (c_type) { - case S32_TYPE: - t = c.s32 >= 0; - break; - case F32_TYPE: - t = c.f32 >= 0; - break; - default: - assert(0); - } + switch ( c_type ) { + case S32_TYPE: t = c.s32 >= 0; break; + case F32_TYPE: t = c.f32 >= 0; break; + default: assert(0); + } - switch (i_type) { - case B16_TYPE: - case S16_TYPE: - case U16_TYPE: - d.u16 = t ? a.u16 : b.u16; - break; - case F32_TYPE: - case B32_TYPE: - case S32_TYPE: - case U32_TYPE: - d.u32 = t ? a.u32 : b.u32; - break; - case F64_TYPE: - case FF64_TYPE: - case B64_TYPE: - case S64_TYPE: - case U64_TYPE: - d.u64 = t ? a.u64 : b.u64; - break; - default: - assert(0); - } + switch ( i_type ) { + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: d.u16 = t?a.u16:b.u16; break; + case F32_TYPE: + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: d.u32 = t?a.u32:b.u32; break; + case F64_TYPE: + case FF64_TYPE: + case B64_TYPE: + case S64_TYPE: + case U64_TYPE: d.u64 = t?a.u64:b.u64; break; + default: assert(0); + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -void sqrt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void sqrt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch (i_type) { - case F32_TYPE: - if (a.f32 < 0) - d.f32 = nanf(""); + + switch ( i_type ) { + case F32_TYPE: + if ( a.f32 < 0 ) + d.f32 = nanf(""); else - d.f32 = sqrt(a.f32); - break; - case F64_TYPE: - case FF64_TYPE: - if (a.f64 < 0) - d.f64 = nan(""); + d.f32 = sqrt(a.f32); break; + case F64_TYPE: + case FF64_TYPE: + if ( a.f64 < 0 ) + d.f64 = nan(""); else - d.f64 = sqrt(a.f64); - break; - default: + d.f64 = sqrt(a.f64); break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst,d, i_type, thread, pI); } -void sst_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_instruction *cpI = const_cast<ptx_instruction *>(pI); // constant - const operand_info &dst = cpI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - unsigned type = pI->get_type(); - ptx_reg_t dst_data = thread->get_operand_value(dst, dst, type, thread, 1); - ptx_reg_t src1_data = thread->get_operand_value(src1, src1, type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, src1, type, thread, 1); - ptx_reg_t src3_data = thread->get_operand_value(src3, src1, type, thread, 1); - memory_space_t space = pI->get_space(); - memory_space *mem = NULL; - addr_t addr = - src2_data.u32 * 4; // this assumes sstarr memory starts at address 0 - ptx_cta_info *cta_info = thread->m_cta_info; +void sst_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_instruction * cpI = const_cast<ptx_instruction *>(pI); // constant + const operand_info &dst = cpI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + unsigned type = pI->get_type(); + ptx_reg_t dst_data = thread->get_operand_value(dst, dst, type, thread, 1); + ptx_reg_t src1_data = thread->get_operand_value(src1, src1, type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, src1, type, thread, 1); + ptx_reg_t src3_data = thread->get_operand_value(src3, src1, type, thread, 1); + memory_space_t space = pI->get_space(); + memory_space *mem = NULL; + addr_t addr = src2_data.u32 * 4; // this assumes sstarr memory starts at address 0 + ptx_cta_info *cta_info = thread->m_cta_info; - decode_space(space, thread, src1, mem, addr); + decode_space(space,thread,src1,mem,addr); - size_t size; - int t; - type_info_key::type_decode(type, size, t); + size_t size; + int t; + type_info_key::type_decode(type,size,t); - // store data in sstarr memory - mem->write(addr, size / 8, &src3_data.s64, thread, pI); + // store data in sstarr memory + mem->write(addr,size/8,&src3_data.s64,thread,pI); - // sync threads - cpI->set_bar_id(16); // use 16 for sst because bar uses an int from 0-15 + // sync threads + cpI->set_bar_id(16); // use 16 for sst because bar uses an int from 0-15 - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; - thread->m_last_dram_callback.function = bar_callback; - thread->m_last_dram_callback.instruction = cpI; + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; + thread->m_last_dram_callback.function = bar_callback; + thread->m_last_dram_callback.instruction = cpI; - // the last thread that executes loads all of the data back from sstarr memory - int NUM_THREADS = cta_info->num_threads(); - cta_info->inc_bar_threads(); - if (NUM_THREADS == cta_info->get_bar_threads()) { - unsigned offset = 0; - addr = 0; - ptx_reg_t data; - float sstarr_fdata[NUM_THREADS]; - signed long long sstarr_ldata[NUM_THREADS]; - // loop through all of the threads - for (int tid = 0; tid < NUM_THREADS; tid++) { - data.u64 = 0; - mem->read(addr + (tid * 4), size / 8, &data.s64); - sstarr_fdata[tid] = data.f32; - sstarr_ldata[tid] = data.s64; - } + // the last thread that executes loads all of the data back from sstarr memory + int NUM_THREADS = cta_info->num_threads(); + cta_info->inc_bar_threads(); + if (NUM_THREADS == cta_info->get_bar_threads()) { + unsigned offset = 0; + addr = 0; + ptx_reg_t data; + float sstarr_fdata[NUM_THREADS]; + signed long long sstarr_ldata[NUM_THREADS]; + // loop through all of the threads + for (int tid = 0; tid < NUM_THREADS; tid++) { + data.u64=0; + mem->read(addr+(tid*4),size/8,&data.s64); + sstarr_fdata[tid] = data.f32; + sstarr_ldata[tid] = data.s64; + } - // squeeze the zeros out of the array and store data back into original - // array - mem = NULL; - addr = src1_data.u32; - space.set_type(global_space); - decode_space(space, thread, src1, mem, addr); - // store nonzero entries and indices - for (int tid = 0; tid < NUM_THREADS; tid++) { - if (sstarr_fdata[tid] != 0) { - float ftid = (float)tid; - mem->write(addr + (offset * 4), size / 8, &sstarr_ldata[tid], thread, - pI); - mem->write(addr + ((NUM_THREADS + offset) * 4), size / 8, &ftid, thread, - pI); - offset++; - } - } - // store the number of nonzero elements in the array - data = thread->get_operand_value(src1, dst, type, thread, 1); - data.s64 += 4 * (offset - 1); - thread->set_operand_value(dst, data, type, thread, pI); + // squeeze the zeros out of the array and store data back into original array + mem = NULL; + addr = src1_data.u32; + space.set_type(global_space); + decode_space(space,thread,src1,mem,addr); + // store nonzero entries and indices + for (int tid = 0; tid < NUM_THREADS; tid++) { + if (sstarr_fdata[tid] != 0) { + float ftid = (float)tid; + mem->write(addr+(offset*4),size/8,&sstarr_ldata[tid],thread,pI); + mem->write(addr+((NUM_THREADS+offset)*4),size/8,&ftid,thread,pI); + offset++; + } + } + // store the number of nonzero elements in the array + data = thread->get_operand_value(src1, dst, type, thread, 1); + data.s64 += 4*(offset-1); + thread->set_operand_value(dst, data, type, thread, pI); - // fill the rest of the array with zeros (dst should always have a 0 in it) - while (offset < NUM_THREADS) { - mem->write(addr + (offset * 4), size / 8, &dst_data.s64, thread, pI); - offset++; - } + // fill the rest of the array with zeros (dst should always have a 0 in it) + while (offset < NUM_THREADS) { + mem->write(addr+(offset*4),size/8,&dst_data.s64,thread,pI); + offset++; + } - cta_info->reset_bar_threads(); - thread->m_last_effective_address = addr + (NUM_THREADS - 1) * 4; - thread->m_last_memory_space = space; - } + cta_info->reset_bar_threads(); + thread->m_last_effective_address = addr+(NUM_THREADS-1)*4; + thread->m_last_memory_space = space; + } } -void ssy_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - // printf("Execution Warning: unimplemented ssy instruction is treated as a - // nop\n"); - // TODO: add implementation +void ssy_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + //printf("Execution Warning: unimplemented ssy instruction is treated as a nop\n"); + // TODO: add implementation } -void st_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); // may be scalar or vector of regs - unsigned type = pI->get_type(); - ptx_reg_t addr_reg = thread->get_operand_value(dst, dst, type, thread, 1); - ptx_reg_t data; - memory_space_t space = pI->get_space(); - unsigned vector_spec = pI->get_vector(); +void st_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); //may be scalar or vector of regs + unsigned type = pI->get_type(); + ptx_reg_t addr_reg = thread->get_operand_value(dst, dst, type, thread, 1); + ptx_reg_t data; + memory_space_t space = pI->get_space(); + unsigned vector_spec = pI->get_vector(); - memory_space *mem = NULL; - addr_t addr = addr_reg.u32; + memory_space *mem = NULL; + addr_t addr = addr_reg.u32; - decode_space(space, thread, dst, mem, addr); + decode_space(space,thread,dst,mem,addr); - size_t size; - int t; - type_info_key::type_decode(type, size, t); + size_t size; + int t; + type_info_key::type_decode(type,size,t); - if (!vector_spec) { - data = thread->get_operand_value(src1, dst, type, thread, 1); - mem->write(addr, size / 8, &data.s64, thread, pI); - } else { - if (vector_spec == V2_TYPE) { - ptx_reg_t *ptx_regs = new ptx_reg_t[2]; - thread->get_vector_operand_values(src1, ptx_regs, 2); - mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); - mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); - delete[] ptx_regs; - } - if (vector_spec == V3_TYPE) { - ptx_reg_t *ptx_regs = new ptx_reg_t[3]; - thread->get_vector_operand_values(src1, ptx_regs, 3); - mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); - mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); - mem->write(addr + 2 * size / 8, size / 8, &ptx_regs[2].s64, thread, pI); - delete[] ptx_regs; - } - if (vector_spec == V4_TYPE) { - ptx_reg_t *ptx_regs = new ptx_reg_t[4]; - thread->get_vector_operand_values(src1, ptx_regs, 4); - mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); - mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); - mem->write(addr + 2 * size / 8, size / 8, &ptx_regs[2].s64, thread, pI); - mem->write(addr + 3 * size / 8, size / 8, &ptx_regs[3].s64, thread, pI); - delete[] ptx_regs; - } - } - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; + if (!vector_spec) { + data = thread->get_operand_value(src1, dst, type, thread, 1); + mem->write(addr,size/8,&data.s64,thread,pI); + } else { + if (vector_spec == V2_TYPE) { + ptx_reg_t* ptx_regs = new ptx_reg_t[2]; + thread->get_vector_operand_values(src1, ptx_regs, 2); + mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); + mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); + delete [] ptx_regs; + } + if (vector_spec == V3_TYPE) { + ptx_reg_t* ptx_regs = new ptx_reg_t[3]; + thread->get_vector_operand_values(src1, ptx_regs, 3); + mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); + mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); + mem->write(addr+2*size/8,size/8,&ptx_regs[2].s64,thread,pI); + delete [] ptx_regs; + } + if (vector_spec == V4_TYPE) { + ptx_reg_t* ptx_regs = new ptx_reg_t[4]; + thread->get_vector_operand_values(src1, ptx_regs, 4); + mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); + mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); + mem->write(addr+2*size/8,size/8,&ptx_regs[2].s64,thread,pI); + mem->write(addr+3*size/8,size/8,&ptx_regs[3].s64,thread,pI); + delete [] ptx_regs; + } + } + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; } -void sub_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t data; - int overflow = 0; - int carry = 0; +void sub_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t data; + int overflow = 0; + int carry = 0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - // performs addition. Sets carry and overflow if needed. - // the constant is added in during subtraction so the carry bit is set - // properly. - switch (i_type) { - case S8_TYPE: + //performs addition. Sets carry and overflow if needed. + //the constant is added in during subtraction so the carry bit is set properly. + switch ( i_type ) { + case S8_TYPE: data.s64 = (src1_data.s64 & 0xFF) - (src2_data.s64 & 0xFF) + 0x100; - if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) != 0) { - overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; - } - carry = (data.s32 & 0x100) >> 8; + if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) != 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } + carry = (data.s32 & 0x100)>>8; break; - case S16_TYPE: + case S16_TYPE: data.s64 = (src1_data.s64 & 0xFFFF) - (src2_data.s64 & 0xFFFF) + 0x10000; - if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) != 0) { - overflow = - ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; - } - carry = (data.s32 & 0x10000) >> 16; + if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) != 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } + carry = (data.s32 & 0x10000)>>16; break; - case S32_TYPE: - data.s64 = (src1_data.s64 & 0xFFFFFFFF) - (src2_data.s64 & 0xFFFFFFFF) + - 0x100000000; - if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) != 0) { - overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 - ? 0 - : 1; - } - carry = ((data.u64) >> 32) & 0x0001; + case S32_TYPE: + data.s64 = (src1_data.s64 & 0xFFFFFFFF) - (src2_data.s64 & 0xFFFFFFFF) + 0x100000000; + if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) != 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } + carry = ((data.u64)>>32) & 0x0001; break; - case S64_TYPE: - data.s64 = src1_data.s64 - src2_data.s64; - break; - case B8_TYPE: - case U8_TYPE: + case S64_TYPE: + data.s64 = src1_data.s64 - src2_data.s64; break; + case B8_TYPE: + case U8_TYPE: data.u64 = (src1_data.u64 & 0xFF) - (src2_data.u64 & 0xFF) + 0x100; - carry = (data.u64 & 0x100) >> 8; + carry = (data.u64 & 0x100)>>8; break; - case B16_TYPE: - case U16_TYPE: + case B16_TYPE: + case U16_TYPE: data.u64 = (src1_data.u64 & 0xFFFF) - (src2_data.u64 & 0xFFFF) + 0x10000; - carry = (data.u64 & 0x10000) >> 16; - break; - case B32_TYPE: - case U32_TYPE: - data.u64 = (src1_data.u64 & 0xFFFFFFFF) - (src2_data.u64 & 0xFFFFFFFF) + - 0x100000000; - carry = (data.u64 & 0x100000000) >> 32; - break; - case B64_TYPE: - case U64_TYPE: - data.u64 = src1_data.u64 - src2_data.u64; - break; - case F16_TYPE: - data.f16 = src1_data.f16 - src2_data.f16; - break; // assert(0); break; - case F32_TYPE: - data.f32 = src1_data.f32 - src2_data.f32; + carry = (data.u64 & 0x10000)>>16; break; - case F64_TYPE: - case FF64_TYPE: - data.f64 = src1_data.f64 - src2_data.f64; + case B32_TYPE: + case U32_TYPE: + data.u64 = (src1_data.u64 & 0xFFFFFFFF) - (src2_data.u64 & 0xFFFFFFFF) + 0x100000000; + carry = (data.u64 & 0x100000000)>>32; break; - default: - assert(0); - break; - } + case B64_TYPE: + case U64_TYPE: + data.u64 = src1_data.u64 - src2_data.u64; break; + case F16_TYPE: data.f16 = src1_data.f16 - src2_data.f16; break;//assert(0); break; + case F32_TYPE: data.f32 = src1_data.f32 - src2_data.f32; break; + case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 - src2_data.f64; break; + default: assert(0); break; + } - thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); + thread->set_operand_value(dst,data, i_type, thread, pI, overflow, carry); } -void nop_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - // Do nothing +void nop_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + // Do nothing } -void subc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void suld_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void sured_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void sust_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void suq_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} +void subc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void suld_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void sured_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void sust_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void suq_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } + union intfloat { - int a; - float b; + int a; + float b; }; -float reduce_precision(float x, unsigned bits) { - intfloat tmp; - tmp.b = x; - int v = tmp.a; - int man = v & ((1 << 23) - 1); - int mask = ((1 << bits) - 1) << (23 - bits); - int nv = (v & ((-1) - ((1 << 23) - 1))) | (mask & man); - tmp.a = nv; - float result = tmp.b; - return result; +float reduce_precision( float x, unsigned bits ) +{ + intfloat tmp; + tmp.b = x; + int v = tmp.a; + int man = v & ((1<<23)-1); + int mask = ((1<<bits)-1) << (23-bits); + int nv = (v & ((-1)-((1<<23)-1))) | (mask&man); + tmp.a = nv; + float result = tmp.b; + return result; } -unsigned wrap(unsigned x, unsigned y, unsigned mx, unsigned my, - size_t elem_size) { - unsigned nx = (mx + x) % mx; - unsigned ny = (my + y) % my; - return nx + mx * ny; +unsigned wrap( unsigned x, unsigned y, unsigned mx, unsigned my, size_t elem_size ) +{ + unsigned nx = (mx+x)%mx; + unsigned ny = (my+y)%my; + return nx + mx*ny; } -unsigned clamp(unsigned x, unsigned y, unsigned mx, unsigned my, - size_t elem_size) { - unsigned nx = x; - while (nx >= mx) nx -= elem_size; - unsigned ny = (y >= my) ? my - 1 : y; - return nx + mx * ny; +unsigned clamp( unsigned x, unsigned y, unsigned mx, unsigned my, size_t elem_size ) +{ + unsigned nx = x; + while (nx >= mx) nx -= elem_size; + unsigned ny = (y >= my)? my - 1 : y; + return nx + mx*ny; } -typedef unsigned (*texAddr_t)(unsigned x, unsigned y, unsigned mx, unsigned my, - size_t elem_size); -float tex_linf_sampling(memory_space *mem, unsigned tex_array_base, int x, - int y, unsigned int width, unsigned int height, - size_t elem_size, float alpha, float beta, - texAddr_t b_lim) { - float Tij; - float Ti1j; - float Tij1; - float Ti1j1; +typedef unsigned (*texAddr_t) (unsigned x, unsigned y, unsigned mx, unsigned my, size_t elem_size); +float tex_linf_sampling(memory_space* mem, unsigned tex_array_base, + int x, int y, unsigned int width, unsigned int height, size_t elem_size, + float alpha, float beta, texAddr_t b_lim) +{ + float Tij; + float Ti1j; + float Tij1; + float Ti1j1; - mem->read(tex_array_base + b_lim(x, y, width, height, elem_size), 4, &Tij); - mem->read(tex_array_base + b_lim(x + elem_size, y, width, height, elem_size), - 4, &Ti1j); - mem->read(tex_array_base + b_lim(x, y + 1, width, height, elem_size), 4, - &Tij1); - mem->read( - tex_array_base + b_lim(x + elem_size, y + 1, width, height, elem_size), 4, - &Ti1j1); + mem->read(tex_array_base + b_lim(x,y,width,height,elem_size), 4, &Tij); + mem->read(tex_array_base + b_lim(x+elem_size,y,width,height,elem_size), 4, &Ti1j); + mem->read(tex_array_base + b_lim(x,y+1,width,height,elem_size), 4, &Tij1); + mem->read(tex_array_base + b_lim(x+elem_size,y+1,width,height,elem_size), 4, &Ti1j1); - float sample = (1 - alpha) * (1 - beta) * Tij + alpha * (1 - beta) * Ti1j + - (1 - alpha) * beta * Tij1 + alpha * beta * Ti1j1; - - return sample; + float sample = (1-alpha)*(1-beta)*Tij + + alpha*(1-beta)*Ti1j + + (1-alpha)*beta*Tij1 + + alpha*beta*Ti1j1; + + return sample; } -float textureNormalizeElementSigned(int element, int bits) { - if (bits) { - int maxN = (1 << bits) - 1; - // removing upper bits - element &= maxN; - // normalizing the number to [-1.0,1.0] - maxN >>= 1; - float output = (float)element / maxN; - if (output < -1.0f) output = -1.0f; - return output; - } else { - return 0.0f; - } +float textureNormalizeElementSigned(int element, int bits) +{ + if (bits) { + int maxN = (1 << bits) - 1; + // removing upper bits + element &= maxN; + // normalizing the number to [-1.0,1.0] + maxN >>= 1; + float output = (float) element / maxN; + if (output < -1.0f) output = -1.0f; + return output; + } else { + return 0.0f; + } } -float textureNormalizeElementUnsigned(unsigned int element, int bits) { - if (bits) { - unsigned int maxN = (1 << bits) - 1; - // removing upper bits and normalizing the number to [0.0,1.0] - return (float)(element & maxN) / maxN; - } else { - return 0.0f; - } +float textureNormalizeElementUnsigned(unsigned int element, int bits) +{ + if (bits) { + unsigned int maxN = (1 << bits) - 1; + // removing upper bits and normalizing the number to [0.0,1.0] + return (float)(element & maxN) / maxN; + } else { + return 0.0f; + } } -void textureNormalizeOutput(const struct cudaChannelFormatDesc &desc, - ptx_reg_t &datax, ptx_reg_t &datay, - ptx_reg_t &dataz, ptx_reg_t &dataw) { - if (desc.f == cudaChannelFormatKindSigned) { - datax.f32 = textureNormalizeElementSigned(datax.s32, desc.x); - datay.f32 = textureNormalizeElementSigned(datay.s32, desc.y); - dataz.f32 = textureNormalizeElementSigned(dataz.s32, desc.z); - dataw.f32 = textureNormalizeElementSigned(dataw.s32, desc.w); - } else if (desc.f == cudaChannelFormatKindUnsigned) { - datax.f32 = textureNormalizeElementUnsigned(datax.u32, desc.x); - datay.f32 = textureNormalizeElementUnsigned(datay.u32, desc.y); - dataz.f32 = textureNormalizeElementUnsigned(dataz.u32, desc.z); - dataw.f32 = textureNormalizeElementUnsigned(dataw.u32, desc.w); - } else { - assert(0 && - "Undefined texture read mode: cudaReadModeNormalizedFloat expect " - "integer elements"); - } +void textureNormalizeOutput( const struct cudaChannelFormatDesc& desc, ptx_reg_t& datax, ptx_reg_t& datay, ptx_reg_t& dataz, ptx_reg_t& dataw ) +{ + if (desc.f == cudaChannelFormatKindSigned) { + datax.f32 = textureNormalizeElementSigned( datax.s32, desc.x ); + datay.f32 = textureNormalizeElementSigned( datay.s32, desc.y ); + dataz.f32 = textureNormalizeElementSigned( dataz.s32, desc.z ); + dataw.f32 = textureNormalizeElementSigned( dataw.s32, desc.w ); + } else if (desc.f == cudaChannelFormatKindUnsigned) { + datax.f32 = textureNormalizeElementUnsigned( datax.u32, desc.x ); + datay.f32 = textureNormalizeElementUnsigned( datay.u32, desc.y ); + dataz.f32 = textureNormalizeElementUnsigned( dataz.u32, desc.z ); + dataw.f32 = textureNormalizeElementUnsigned( dataw.u32, desc.w ); + } else { + assert(0 && "Undefined texture read mode: cudaReadModeNormalizedFloat expect integer elements"); + } } -void tex_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - unsigned dimension = pI->dimension(); - const operand_info &dst = - pI->dst(); // the registers to which fetched texel will be placed - const operand_info &src1 = pI->src1(); // the name of the texture - const operand_info &src2 = pI->src2(); // the vector registers containing - // coordinates of the texel to be - // fetched +void tex_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + unsigned dimension = pI->dimension(); + const operand_info &dst = pI->dst(); //the registers to which fetched texel will be placed + const operand_info &src1 = pI->src1(); //the name of the texture + const operand_info &src2 = pI->src2(); //the vector registers containing coordinates of the texel to be fetched - std::string texname = src1.name(); - unsigned to_type = pI->get_type(); - unsigned c_type = pI->get_type2(); - fflush(stdout); - ptx_reg_t data1, data2, data3, data4; - if (!thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs) - thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs = new ptx_reg_t[4]; - unsigned nelem = src2.get_vect_nelem(); - thread->get_vector_operand_values( - src2, thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs, - nelem); // ptx_reg should be 4 entry vector type...coordinates into - // texture - /* - For programs with many streams, textures can be bound and unbound - asynchronously. This means we need to use the kernel's "snapshot" of - the state of the texture mappings when it was launched (so that we - don't try to access the incorrect texture mapping if it's been updated, - or that we don't access a mapping that has been unbound). - */ - gpgpu_t *gpu = thread->get_gpu(); - kernel_info_t &k = thread->get_kernel(); - const struct textureReference *texref = gpu->get_texref(texname); - const struct cudaArray *cuArray = k.get_texarray(texname); - const struct textureInfo *texInfo = k.get_texinfo(texname); - const struct textureReferenceAttr *texAttr = gpu->get_texattr(texname); + std::string texname = src1.name(); + unsigned to_type = pI->get_type(); + unsigned c_type = pI->get_type2(); + fflush(stdout); + ptx_reg_t data1, data2, data3, data4; + if (!thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs) + thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs = new ptx_reg_t[4]; + unsigned nelem = src2.get_vect_nelem(); + thread->get_vector_operand_values(src2, thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs, nelem); //ptx_reg should be 4 entry vector type...coordinates into texture + /* + For programs with many streams, textures can be bound and unbound + asynchronously. This means we need to use the kernel's "snapshot" of + the state of the texture mappings when it was launched (so that we + don't try to access the incorrect texture mapping if it's been updated, + or that we don't access a mapping that has been unbound). + */ + gpgpu_t *gpu = thread->get_gpu(); + kernel_info_t &k = thread->get_kernel(); + const struct textureReference* texref = gpu->get_texref(texname); + const struct cudaArray* cuArray = k.get_texarray(texname); + const struct textureInfo* texInfo = k.get_texinfo(texname); + const struct textureReferenceAttr* texAttr = gpu->get_texattr(texname); - // assume always 2D f32 input - // access array with src2 coordinates - memory_space *mem = thread->get_global_memory(); - float x_f32, y_f32; - size_t size; - int t; - unsigned tex_array_base; - unsigned int width = 0, height = 0; - int x = 0; - int y = 0; - unsigned tex_array_index; - float alpha = 0, beta = 0; + //assume always 2D f32 input + //access array with src2 coordinates + memory_space *mem = thread->get_global_memory(); + float x_f32, y_f32; + size_t size; + int t; + unsigned tex_array_base; + unsigned int width = 0, height = 0; + int x = 0; + int y = 0; + unsigned tex_array_index; + float alpha=0, beta=0; - type_info_key::type_decode(to_type, size, t); - tex_array_base = cuArray->devPtr32; + type_info_key::type_decode(to_type,size,t); + tex_array_base = cuArray->devPtr32; - switch (dimension) { - case GEOM_MODIFIER_1D: + switch (dimension) { + case GEOM_MODIFIER_1D: width = cuArray->width; height = cuArray->height; if (texref->normalized) { - assert(c_type == F32_TYPE); - x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - if (texref->addressMode[0] == cudaAddressModeClamp) { - x_f32 = (x_f32 > 1.0) ? 1.0 : x_f32; - x_f32 = (x_f32 < 0.0) ? 0.0 : x_f32; - } else if (texref->addressMode[0] == cudaAddressModeWrap) { - x_f32 = x_f32 - floor(x_f32); - } + assert(c_type == F32_TYPE); + x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; + if (texref->addressMode[0] == cudaAddressModeClamp) { + x_f32 = (x_f32 > 1.0)? 1.0 : x_f32; + x_f32 = (x_f32 < 0.0)? 0.0 : x_f32; + } else if (texref->addressMode[0] == cudaAddressModeWrap) { + x_f32 = x_f32 - floor(x_f32); + } - if (texref->filterMode == cudaFilterModeLinear) { - float xb = x_f32 * width - 0.5; - alpha = xb - floor(xb); - alpha = reduce_precision(alpha, 9); - beta = 0.0; + if( texref->filterMode == cudaFilterModeLinear ) { + float xb = x_f32 * width - 0.5; + alpha = xb - floor(xb); + alpha = reduce_precision(alpha,9); + beta = 0.0; - x = (int)floor(xb); - y = 0; - } else { - x = (int)floor(x_f32 * width); - y = 0; - } + x = (int)floor(xb); + y = 0; + } else { + x = (int) floor(x_f32 * width); + y = 0; + } } else { - switch (c_type) { - case S32_TYPE: + switch ( c_type ) { + case S32_TYPE: x = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].s32; - assert(texref->filterMode == cudaFilterModePoint); - break; - case F32_TYPE: + assert(texref->filterMode == cudaFilterModePoint); + break; + case F32_TYPE: x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - alpha = x_f32 - - floor(x_f32); // offset into subtexel (for linear sampling) - x = (int)x_f32; - break; - default: - assert(0 && "Unsupported texture coordinate type."); - } - // handle texture fetch that exceeded boundaries - if (texref->addressMode[0] == cudaAddressModeClamp) { - x = (x > width - 1) ? (width - 1) : x; - x = (x < 0) ? 0 : x; - } else if (texref->addressMode[0] == cudaAddressModeWrap) { - x = x % width; - } + alpha = x_f32 - floor(x_f32); // offset into subtexel (for linear sampling) + x = (int) x_f32; + break; + default: assert(0 && "Unsupported texture coordinate type."); + } + // handle texture fetch that exceeded boundaries + if (texref->addressMode[0] == cudaAddressModeClamp) { + x = (x > width - 1)? (width - 1) : x; + x = (x < 0)? 0 : x; + } else if (texref->addressMode[0] == cudaAddressModeWrap) { + x = x % width; + } } - width *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + - cuArray->desc.z) / - 8; - x *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + - cuArray->desc.z) / - 8; + width *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; + x *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; tex_array_index = tex_array_base + x; break; - case GEOM_MODIFIER_2D: + case GEOM_MODIFIER_2D: width = cuArray->width; height = cuArray->height; if (texref->normalized) { - x_f32 = reduce_precision( - thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32, 16); - y_f32 = reduce_precision( - thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32, 15); + x_f32 = reduce_precision(thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32,16); + y_f32 = reduce_precision(thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32,15); - if (texref->addressMode[0]) { // clamp - if (x_f32 < 0) x_f32 = 0; - if (x_f32 >= 1) x_f32 = 1 - 1 / x_f32; - } else { // wrap - x_f32 = x_f32 - floor(x_f32); - } - if (texref->addressMode[1]) { // clamp - if (y_f32 < 0) y_f32 = 0; - if (y_f32 >= 1) y_f32 = 1 - 1 / y_f32; - } else { // wrap - y_f32 = y_f32 - floor(y_f32); - } + if (texref->addressMode[0]) {//clamp + if (x_f32<0) x_f32 = 0; + if (x_f32>=1) x_f32 = 1 - 1/x_f32; + } else {//wrap + x_f32 = x_f32 - floor(x_f32); + } + if (texref->addressMode[1]) {//clamp + if (y_f32<0) y_f32 = 0; + if (y_f32>=1) y_f32 = 1 - 1/y_f32; + } else {//wrap + y_f32 = y_f32 - floor(y_f32); + } - if (texref->filterMode == cudaFilterModeLinear) { - float xb = x_f32 * width - 0.5; - float yb = y_f32 * height - 0.5; - alpha = xb - floor(xb); - beta = yb - floor(yb); - alpha = reduce_precision(alpha, 9); - beta = reduce_precision(beta, 9); + if( texref->filterMode == cudaFilterModeLinear ) { + float xb = x_f32 * width - 0.5; + float yb = y_f32 * height - 0.5; + alpha = xb - floor(xb); + beta = yb - floor(yb); + alpha = reduce_precision(alpha,9); + beta = reduce_precision(beta,9); - x = (int)floor(xb); - y = (int)floor(yb); - } else { - x = (int)floor(x_f32 * width); - y = (int)floor(y_f32 * height); - } + x = (int)floor(xb); + y = (int)floor(yb); + } else { + x = (int) floor(x_f32 * width); + y = (int) floor(y_f32 * height); + } } else { - x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - y_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32; + x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; + y_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32; - alpha = x_f32 - floor(x_f32); - beta = y_f32 - floor(y_f32); + alpha = x_f32 - floor(x_f32); + beta = y_f32 - floor(y_f32); - x = (int)x_f32; - y = (int)y_f32; - if (texref->addressMode[0]) { // clamp - if (x < 0) x = 0; - if (x >= (int)width) x = width - 1; - } else { // wrap - x = x % width; - if (x < 0) x *= -1; - } - if (texref->addressMode[1]) { // clamp - if (y < 0) y = 0; - if (y >= (int)height) y = height - 1; - } else { // wrap - y = y % height; - if (y < 0) y *= -1; - } + x = (int) x_f32; + y = (int) y_f32; + if (texref->addressMode[0]) {//clamp + if (x<0) x = 0; + if (x>= (int)width) x = width-1; + } else {//wrap + x = x % width; + if (x < 0) x*= -1; + } + if (texref->addressMode[1]) {//clamp + if (y<0) y = 0; + if (y>= (int)height) y = height -1; + } else {//wrap + y = y % height; + if (y < 0) y *= -1; + } } - width *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + - cuArray->desc.z) / - 8; - x *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + - cuArray->desc.z) / - 8; - tex_array_index = tex_array_base + (x + width * y); + width *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; + x *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; + tex_array_index = tex_array_base + (x + width*y); break; - default: - assert(0); - break; - } - switch (to_type) { - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case B8_TYPE: - case B16_TYPE: - case B32_TYPE: - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: { - unsigned long long elementOffset = 0; // offset into the next element - mem->read(tex_array_index, cuArray->desc.x / 8, &data1.u32); - elementOffset += cuArray->desc.x / 8; + default: + assert(0); break; + } + switch ( to_type ) { + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case B8_TYPE: + case B16_TYPE: + case B32_TYPE: + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: { + unsigned long long elementOffset = 0; // offset into the next element + mem->read( tex_array_index, cuArray->desc.x/8, &data1.u32); + elementOffset += cuArray->desc.x/8; if (cuArray->desc.y) { - mem->read(tex_array_index + elementOffset, cuArray->desc.y / 8, - &data2.u32); - elementOffset += cuArray->desc.y / 8; - if (cuArray->desc.z) { - mem->read(tex_array_index + elementOffset, cuArray->desc.z / 8, - &data3.u32); - elementOffset += cuArray->desc.z / 8; - if (cuArray->desc.w) - mem->read(tex_array_index + elementOffset, cuArray->desc.w / 8, - &data4.u32); - } + mem->read( tex_array_index + elementOffset, cuArray->desc.y/8, &data2.u32); + elementOffset += cuArray->desc.y/8; + if (cuArray->desc.z) { + mem->read( tex_array_index + elementOffset, cuArray->desc.z/8, &data3.u32); + elementOffset += cuArray->desc.z/8; + if (cuArray->desc.w) + mem->read( tex_array_index + elementOffset, cuArray->desc.w/8, &data4.u32); + } } break; - } - case B64_TYPE: - case U64_TYPE: - case S64_TYPE: - mem->read(tex_array_index, 8, &data1.u64); + } + case B64_TYPE: + case U64_TYPE: + case S64_TYPE: + mem->read( tex_array_index, 8, &data1.u64); if (cuArray->desc.y) { - mem->read(tex_array_index + 8, 8, &data2.u64); - if (cuArray->desc.z) { - mem->read(tex_array_index + 16, 8, &data3.u64); - if (cuArray->desc.w) mem->read(tex_array_index + 24, 8, &data4.u64); - } + mem->read( tex_array_index+8, 8, &data2.u64); + if (cuArray->desc.z) { + mem->read( tex_array_index+16, 8, &data3.u64); + if (cuArray->desc.w) + mem->read( tex_array_index+24, 8, &data4.u64); + } } break; - case F16_TYPE: - assert(0); - break; - case F32_TYPE: { - if (texref->filterMode == cudaFilterModeLinear) { - texAddr_t b_lim = wrap; - if (texref->addressMode[0] == cudaAddressModeClamp) { - b_lim = clamp; - } - size_t elem_size = (cuArray->desc.x + cuArray->desc.y + - cuArray->desc.z + cuArray->desc.w) / - 8; - size_t elem_ofst = 0; + case F16_TYPE: assert(0); break; + case F32_TYPE: { + if( texref->filterMode == cudaFilterModeLinear ) { + texAddr_t b_lim = wrap; + if ( texref->addressMode[0] == cudaAddressModeClamp ) { + b_lim = clamp; + } + size_t elem_size = (cuArray->desc.x + cuArray->desc.y + cuArray->desc.z + cuArray->desc.w) / 8; + size_t elem_ofst = 0; - data1.f32 = - tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, - height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.x / 8; - if (cuArray->desc.y) { - data2.f32 = - tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, - height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.y / 8; - if (cuArray->desc.z) { - data3.f32 = - tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, - height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.z / 8; - if (cuArray->desc.w) - data4.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, - y, width, height, elem_size, alpha, - beta, b_lim); - } - } + data1.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.x / 8; + if (cuArray->desc.y) { + data2.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.y / 8; + if (cuArray->desc.z) { + data3.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.z / 8; + if (cuArray->desc.w) + data4.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); + } + } } else { - mem->read(tex_array_index, cuArray->desc.x / 8, &data1.f32); - if (cuArray->desc.y) { - mem->read(tex_array_index + 4, cuArray->desc.y / 8, &data2.f32); - if (cuArray->desc.z) { - mem->read(tex_array_index + 8, cuArray->desc.z / 8, &data3.f32); - if (cuArray->desc.w) - mem->read(tex_array_index + 12, cuArray->desc.w / 8, &data4.f32); - } - } + mem->read( tex_array_index, cuArray->desc.x/8, &data1.f32); + if (cuArray->desc.y) { + mem->read( tex_array_index+4, cuArray->desc.y/8, &data2.f32); + if (cuArray->desc.z) { + mem->read( tex_array_index+8, cuArray->desc.z/8, &data3.f32); + if (cuArray->desc.w) + mem->read( tex_array_index+12, cuArray->desc.w/8, &data4.f32); + } + } } - } break; - case F64_TYPE: - case FF64_TYPE: - mem->read(tex_array_index, 8, &data1.f64); + } break; + case F64_TYPE: + case FF64_TYPE: + mem->read( tex_array_index, 8, &data1.f64); if (cuArray->desc.y) { - mem->read(tex_array_index + 8, 8, &data2.f64); - if (cuArray->desc.z) { - mem->read(tex_array_index + 16, 8, &data3.f64); - if (cuArray->desc.w) mem->read(tex_array_index + 24, 8, &data4.f64); - } + mem->read( tex_array_index+8, 8, &data2.f64); + if (cuArray->desc.z) { + mem->read( tex_array_index+16, 8, &data3.f64); + if (cuArray->desc.w) + mem->read( tex_array_index+24, 8, &data4.f64); + } } break; - default: - assert(0); - break; - } - int x_block_coord, y_block_coord, memreqindex, blockoffset; + default: assert(0); break; + } + int x_block_coord, y_block_coord, memreqindex, blockoffset; - switch (dimension) { - case GEOM_MODIFIER_1D: + switch (dimension) { + case GEOM_MODIFIER_1D: thread->m_last_effective_address = tex_array_index; break; - case GEOM_MODIFIER_2D: + case GEOM_MODIFIER_2D: x_block_coord = x >> (texInfo->Tx_numbits + texInfo->texel_size_numbits); y_block_coord = y >> texInfo->Ty_numbits; - memreqindex = - ((y_block_coord * cuArray->width / texInfo->Tx) + x_block_coord) << 6; + memreqindex = ((y_block_coord*cuArray->width/texInfo->Tx)+x_block_coord)<<6; - blockoffset = (x % (texInfo->Tx * texInfo->texel_size) + - (y % (texInfo->Ty) - << (texInfo->Tx_numbits + texInfo->texel_size_numbits))); + blockoffset = (x%(texInfo->Tx*texInfo->texel_size) + (y%(texInfo->Ty)<<(texInfo->Tx_numbits + texInfo->texel_size_numbits))); memreqindex += blockoffset; - thread->m_last_effective_address = - tex_array_base + memreqindex; // tex_array_index; + thread->m_last_effective_address = tex_array_base + memreqindex;//tex_array_index; break; - default: + default: assert(0); - } - thread->m_last_memory_space = tex_space; + } + thread->m_last_memory_space = tex_space; - // normalize output into floating point numbers according to the texture read - // mode - if (texAttr->m_readmode == cudaReadModeNormalizedFloat) { - textureNormalizeOutput(cuArray->desc, data1, data2, data3, data4); - } else { - assert(texAttr->m_readmode == cudaReadModeElementType); - } + // normalize output into floating point numbers according to the texture read mode + if (texAttr->m_readmode == cudaReadModeNormalizedFloat) { + textureNormalizeOutput(cuArray->desc, data1, data2, data3, data4); + } else { + assert(texAttr->m_readmode == cudaReadModeElementType); + } - thread->set_vector_operand_values(dst, data1, data2, data3, data4); + thread->set_vector_operand_values(dst,data1,data2,data3,data4); } -void txq_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void trap_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vabsdiff_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vadd_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vmad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vmax_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vmin_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vset_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vshl_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vshr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} -void vsub_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); -} +void txq_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void trap_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vabsdiff_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vadd_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vmad_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vmax_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vmin_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vset_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vshl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vshr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vsub_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vote_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - static bool first_in_warp = true; - static bool and_all; - static bool or_all; - static unsigned int ballot_result; - static std::list<ptx_thread_info *> threads_in_warp; - static unsigned last_tid; +void vote_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + static bool first_in_warp = true; + static bool and_all; + static bool or_all; + static unsigned int ballot_result; + static std::list<ptx_thread_info*> threads_in_warp; + static unsigned last_tid; - if (first_in_warp) { - first_in_warp = false; - threads_in_warp.clear(); - and_all = true; - or_all = false; - ballot_result = 0; - int offset = 31; - while ((offset >= 0) && !pI->active(offset)) offset--; - assert(offset >= 0); - last_tid = - (thread->get_hw_tid() - (thread->get_hw_tid() % pI->warp_size())) + - offset; - } + if( first_in_warp ) { + first_in_warp = false; + threads_in_warp.clear(); + and_all = true; + or_all = false; + ballot_result = 0; + int offset=31; + while( (offset>=0) && !pI->active(offset) ) + offset--; + assert( offset >= 0 ); + last_tid = (thread->get_hw_tid() - (thread->get_hw_tid()%pI->warp_size())) + offset; + } - ptx_reg_t src1_data; - const operand_info &src1 = pI->src1(); - src1_data = thread->get_operand_value(src1, pI->dst(), PRED_TYPE, thread, 1); + ptx_reg_t src1_data; + const operand_info &src1 = pI->src1(); + src1_data = thread->get_operand_value(src1, pI->dst(), PRED_TYPE, thread, 1); - // predicate value was changed so the lowest bit being set means the zero flag - // is set. - // As a result, the value of src1_data.pred must be inverted to get proper - // behavior - bool pred_value = !(src1_data.pred & 0x0001); - bool invert = src1.is_neg_pred(); + //predicate value was changed so the lowest bit being set means the zero flag is set. + //As a result, the value of src1_data.pred must be inverted to get proper behavior + bool pred_value = !(src1_data.pred & 0x0001); + bool invert = src1.is_neg_pred(); - threads_in_warp.push_back(thread); - and_all &= (invert ^ pred_value); - or_all |= (invert ^ pred_value); + threads_in_warp.push_back(thread); + and_all &= (invert ^ pred_value); + or_all |= (invert ^ pred_value); - // vote.ballot - if (invert ^ pred_value) { - int lane_id = thread->get_hw_tid() % pI->warp_size(); - ballot_result |= (1 << lane_id); - } + // vote.ballot + if (invert ^ pred_value) { + int lane_id = thread->get_hw_tid() % pI->warp_size(); + ballot_result |= (1 << lane_id); + } - if (thread->get_hw_tid() == last_tid) { - if (pI->vote_mode() == ptx_instruction::vote_ballot) { - ptx_reg_t data = ballot_result; - for (std::list<ptx_thread_info *>::iterator t = threads_in_warp.begin(); - t != threads_in_warp.end(); ++t) { - const operand_info &dst = pI->dst(); - (*t)->set_operand_value(dst, data, pI->get_type(), (*t), pI); - } - } else { - bool pred_value = false; + if( thread->get_hw_tid() == last_tid ) { + if (pI->vote_mode() == ptx_instruction::vote_ballot) { + ptx_reg_t data = ballot_result; + for( std::list<ptx_thread_info*>::iterator t=threads_in_warp.begin(); t!=threads_in_warp.end(); ++t ) { + const operand_info &dst = pI->dst(); + (*t)->set_operand_value(dst,data, pI->get_type(), (*t), pI); + } + } else { + bool pred_value = false; - switch (pI->vote_mode()) { - case ptx_instruction::vote_any: - pred_value = or_all; - break; - case ptx_instruction::vote_all: - pred_value = and_all; - break; - case ptx_instruction::vote_uni: - pred_value = (or_all ^ and_all); - break; - default: - abort(); - } - ptx_reg_t data; - data.pred = pred_value ? 0 : 1; // the way ptxplus handles the zero flag, - // 1 = false and 0 = true + switch( pI->vote_mode() ) { + case ptx_instruction::vote_any: pred_value = or_all; break; + case ptx_instruction::vote_all: pred_value = and_all; break; + case ptx_instruction::vote_uni: pred_value = (or_all ^ and_all); break; + default: + abort(); + } + ptx_reg_t data; + data.pred = pred_value?0:1; //the way ptxplus handles the zero flag, 1 = false and 0 = true - for (std::list<ptx_thread_info *>::iterator t = threads_in_warp.begin(); - t != threads_in_warp.end(); ++t) { - const operand_info &dst = pI->dst(); - (*t)->set_operand_value(dst, data, PRED_TYPE, (*t), pI); + for( std::list<ptx_thread_info*>::iterator t=threads_in_warp.begin(); t!=threads_in_warp.end(); ++t ) { + const operand_info &dst = pI->dst(); + (*t)->set_operand_value(dst,data, PRED_TYPE, (*t), pI); + } } - } - first_in_warp = true; - } + first_in_warp = true; + } } -void xor_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - ptx_reg_t src1_data, src2_data, data; +void xor_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - // the way ptxplus handles predicates: 1 = false and 0 = true - if (i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) ^ ~(src2_data.pred)); - else - data.u64 = src1_data.u64 ^ src2_data.u64; + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) ^ ~(src2_data.pred)); + else + data.u64 = src1_data.u64 ^ src2_data.u64; - thread->set_operand_value(dst, data, i_type, thread, pI); + thread->set_operand_value(dst,data, i_type, thread, pI); } -void inst_not_implemented(const ptx_instruction *pI) { - printf( - "GPGPU-Sim PTX: ERROR (%s:%u) instruction \"%s\" not (yet) implemented\n", - pI->source_file(), pI->source_line(), pI->get_opcode_cstr()); - abort(); +void inst_not_implemented( const ptx_instruction * pI ) +{ + printf("GPGPU-Sim PTX: ERROR (%s:%u) instruction \"%s\" not (yet) implemented\n", + pI->source_file(), + pI->source_line(), + pI->get_opcode_cstr() ); + abort(); } -ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, - operand_info dstInfo, unsigned type, - ptx_thread_info *thread) { - ptx_reg_t result; - memory_space *mem = NULL; - size_t size; - int t; - result.u64 = 0; +ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread) +{ + ptx_reg_t result; + memory_space *mem = NULL; + size_t size; + int t; + result.u64=0; - // complete other cases for reading from memory, such as reading from other - // const memory - if (opInfo.get_addr_space() == global_space) { - mem = thread->get_global_memory(); - type_info_key::type_decode(type, size, t); - mem->read(opData.u32, size / 8, &result.u64); - if (type == S16_TYPE || type == S32_TYPE) - sign_extend(result, size, dstInfo); - } else if (opInfo.get_addr_space() == shared_space) { - mem = thread->m_shared_mem; - type_info_key::type_decode(type, size, t); - mem->read(opData.u32, size / 8, &result.u64); + //complete other cases for reading from memory, such as reading from other const memory + if(opInfo.get_addr_space() == global_space) + { + mem = thread->get_global_memory(); + type_info_key::type_decode(type,size,t); + mem->read(opData.u32,size/8,&result.u64); + if( type == S16_TYPE || type == S32_TYPE ) + sign_extend(result,size,dstInfo); + } + else if(opInfo.get_addr_space() == shared_space) + { + mem = thread->m_shared_mem; + type_info_key::type_decode(type,size,t); + mem->read(opData.u32,size/8,&result.u64); - if (type == S16_TYPE || type == S32_TYPE) - sign_extend(result, size, dstInfo); + if( type == S16_TYPE || type == S32_TYPE ) + sign_extend(result,size,dstInfo); - } else if (opInfo.get_addr_space() == const_space) { - mem = thread->get_global_memory(); - type_info_key::type_decode(type, size, t); + } + else if(opInfo.get_addr_space() == const_space) + { + mem = thread->get_global_memory(); + type_info_key::type_decode(type,size,t); - mem->read((opData.u32 + opInfo.get_const_mem_offset()), size / 8, - &result.u64); + mem->read((opData.u32 + opInfo.get_const_mem_offset()),size/8,&result.u64); - if (type == S16_TYPE || type == S32_TYPE) - sign_extend(result, size, dstInfo); - } else { - result = opData; - } + if( type == S16_TYPE || type == S32_TYPE ) + sign_extend(result,size,dstInfo); + } + else + { + result = opData; + } - if (opInfo.get_operand_lohi() == 1) { - result.u64 = result.u64 & 0xFFFF; - } else if (opInfo.get_operand_lohi() == 2) { - result.u64 = (result.u64 >> 16) & 0xFFFF; - } + if(opInfo.get_operand_lohi() == 1) + { + result.u64 = result.u64 & 0xFFFF; + } + else if(opInfo.get_operand_lohi() == 2) + { + result.u64 = (result.u64>>16) & 0xFFFF; + } - if (opInfo.get_operand_neg() == true) { - result.f32 = -result.f32; - } + if(opInfo.get_operand_neg() == true) { + result.f32 = -result.f32; + } - return result; + return result; } + diff --git a/src/cuda-sim/memory.cc b/src/cuda-sim/memory.cc index b630da9..4b2acdf 100644 --- a/src/cuda-sim/memory.cc +++ b/src/cuda-sim/memory.cc @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -29,206 +27,190 @@ #include "memory.h" #include <stdlib.h> -#include "../../libcuda/gpgpu_context.h" #include "../debug.h" +#include "../../libcuda/gpgpu_context.h" -template <unsigned BSIZE> -memory_space_impl<BSIZE>::memory_space_impl(std::string name, - unsigned hash_size) { - m_name = name; - MEM_MAP_RESIZE(hash_size); +template<unsigned BSIZE> memory_space_impl<BSIZE>::memory_space_impl( std::string name, unsigned hash_size ) +{ + m_name = name; + MEM_MAP_RESIZE(hash_size); - m_log2_block_size = -1; - for (unsigned n = 0, mask = 1; mask != 0; mask <<= 1, n++) { - if (BSIZE & mask) { - assert(m_log2_block_size == (unsigned)-1); - m_log2_block_size = n; - } - } - assert(m_log2_block_size != (unsigned)-1); + m_log2_block_size = -1; + for( unsigned n=0, mask=1; mask != 0; mask <<= 1, n++ ) { + if( BSIZE & mask ) { + assert( m_log2_block_size == (unsigned)-1 ); + m_log2_block_size = n; + } + } + assert( m_log2_block_size != (unsigned)-1 ); } -template <unsigned BSIZE> -void memory_space_impl<BSIZE>::write_only(mem_addr_t offset, mem_addr_t index, - size_t length, const void *data) { - m_data[index].write(offset, length, (const unsigned char *)data); +template<unsigned BSIZE> void memory_space_impl<BSIZE>::write_only( mem_addr_t offset, mem_addr_t index, size_t length, const void *data) +{ + m_data[index].write(offset,length,(const unsigned char*)data); } -template <unsigned BSIZE> -void memory_space_impl<BSIZE>::write(mem_addr_t addr, size_t length, - const void *data, - class ptx_thread_info *thd, - const ptx_instruction *pI) { - mem_addr_t index = addr >> m_log2_block_size; +template<unsigned BSIZE> void memory_space_impl<BSIZE>::write( mem_addr_t addr, size_t length, const void *data, class ptx_thread_info *thd, const ptx_instruction *pI) +{ - if ((addr + length) <= (index + 1) * BSIZE) { - // fast route for intra-block access - unsigned offset = addr & (BSIZE - 1); - unsigned nbytes = length; - m_data[index].write(offset, nbytes, (const unsigned char *)data); - } else { - // slow route for inter-block access - unsigned nbytes_remain = length; - unsigned src_offset = 0; - mem_addr_t current_addr = addr; + mem_addr_t index = addr >> m_log2_block_size; - while (nbytes_remain > 0) { - unsigned offset = current_addr & (BSIZE - 1); - mem_addr_t page = current_addr >> m_log2_block_size; - mem_addr_t access_limit = offset + nbytes_remain; - if (access_limit > BSIZE) { - access_limit = BSIZE; - } + if ( (addr+length) <= (index+1)*BSIZE ) { + // fast route for intra-block access + unsigned offset = addr & (BSIZE-1); + unsigned nbytes = length; + m_data[index].write(offset,nbytes,(const unsigned char*)data); + } else { + // slow route for inter-block access + unsigned nbytes_remain = length; + unsigned src_offset = 0; + mem_addr_t current_addr = addr; - size_t tx_bytes = access_limit - offset; - m_data[page].write(offset, tx_bytes, - &((const unsigned char *)data)[src_offset]); + while (nbytes_remain > 0) { + unsigned offset = current_addr & (BSIZE-1); + mem_addr_t page = current_addr >> m_log2_block_size; + mem_addr_t access_limit = offset + nbytes_remain; + if (access_limit > BSIZE) { + access_limit = BSIZE; + } + + size_t tx_bytes = access_limit - offset; + m_data[page].write(offset, tx_bytes, &((const unsigned char*)data)[src_offset]); - // advance pointers - src_offset += tx_bytes; - current_addr += tx_bytes; - nbytes_remain -= tx_bytes; - } - assert(nbytes_remain == 0); - } - if (!m_watchpoints.empty()) { - std::map<unsigned, mem_addr_t>::iterator i; - for (i = m_watchpoints.begin(); i != m_watchpoints.end(); i++) { - mem_addr_t wa = i->second; - if (((addr <= wa) && ((addr + length) > wa)) || - ((addr > wa) && (addr < (wa + 4)))) - thd->get_gpu()->gpgpu_ctx->the_gpgpusim->g_the_gpu->hit_watchpoint( - i->first, thd, pI); - } - } + // advance pointers + src_offset += tx_bytes; + current_addr += tx_bytes; + nbytes_remain -= tx_bytes; + } + assert(nbytes_remain == 0); + } + if( !m_watchpoints.empty() ) { + std::map<unsigned,mem_addr_t>::iterator i; + for( i=m_watchpoints.begin(); i!=m_watchpoints.end(); i++ ) { + mem_addr_t wa = i->second; + if( ((addr<=wa) && ((addr+length)>wa)) || ((addr>wa) && (addr < (wa+4))) ) + thd->get_gpu()->gpgpu_ctx->the_gpgpusim->g_the_gpu->hit_watchpoint(i->first,thd,pI); + } + } } -template <unsigned BSIZE> -void memory_space_impl<BSIZE>::read_single_block(mem_addr_t blk_idx, - mem_addr_t addr, size_t length, - void *data) const { - if ((addr + length) > (blk_idx + 1) * BSIZE) { - printf( - "GPGPU-Sim PTX: ERROR * access to memory \'%s\' is unaligned : " - "addr=0x%x, length=%zu\n", - m_name.c_str(), addr, length); - printf( - "GPGPU-Sim PTX: (addr+length)=0x%lx > 0x%x=(index+1)*BSIZE, " - "index=0x%x, BSIZE=0x%x\n", - (addr + length), (blk_idx + 1) * BSIZE, blk_idx, BSIZE); - throw 1; - } - typename map_t::const_iterator i = m_data.find(blk_idx); - if (i == m_data.end()) { - for (size_t n = 0; n < length; n++) - ((unsigned char *)data)[n] = (unsigned char)0; - // printf("GPGPU-Sim PTX: WARNING reading %zu bytes from unititialized - // memory at address 0x%x in space %s\n", length, addr, m_name.c_str() ); - } else { - unsigned offset = addr & (BSIZE - 1); - unsigned nbytes = length; - i->second.read(offset, nbytes, (unsigned char *)data); - } +template<unsigned BSIZE> void memory_space_impl<BSIZE>::read_single_block( mem_addr_t blk_idx, mem_addr_t addr, size_t length, void *data) const +{ + if ((addr + length) > (blk_idx + 1) * BSIZE) { + printf("GPGPU-Sim PTX: ERROR * access to memory \'%s\' is unaligned : addr=0x%x, length=%zu\n", + m_name.c_str(), addr, length); + printf("GPGPU-Sim PTX: (addr+length)=0x%lx > 0x%x=(index+1)*BSIZE, index=0x%x, BSIZE=0x%x\n", + (addr+length),(blk_idx+1)*BSIZE, blk_idx, BSIZE); + throw 1; + } + typename map_t::const_iterator i = m_data.find(blk_idx); + if( i == m_data.end() ) { + for( size_t n=0; n < length; n++ ) + ((unsigned char*)data)[n] = (unsigned char) 0; + //printf("GPGPU-Sim PTX: WARNING reading %zu bytes from unititialized memory at address 0x%x in space %s\n", length, addr, m_name.c_str() ); + } else { + unsigned offset = addr & (BSIZE-1); + unsigned nbytes = length; + i->second.read(offset,nbytes,(unsigned char*)data); + } } -template <unsigned BSIZE> -void memory_space_impl<BSIZE>::read(mem_addr_t addr, size_t length, - void *data) const { - mem_addr_t index = addr >> m_log2_block_size; - if ((addr + length) <= (index + 1) * BSIZE) { - // fast route for intra-block access - read_single_block(index, addr, length, data); - } else { - // slow route for inter-block access - unsigned nbytes_remain = length; - unsigned dst_offset = 0; - mem_addr_t current_addr = addr; +template<unsigned BSIZE> void memory_space_impl<BSIZE>::read( mem_addr_t addr, size_t length, void *data ) const +{ + mem_addr_t index = addr >> m_log2_block_size; + if ((addr+length) <= (index+1)*BSIZE ) { + // fast route for intra-block access + read_single_block(index, addr, length, data); + } else { + // slow route for inter-block access + unsigned nbytes_remain = length; + unsigned dst_offset = 0; + mem_addr_t current_addr = addr; - while (nbytes_remain > 0) { - unsigned offset = current_addr & (BSIZE - 1); - mem_addr_t page = current_addr >> m_log2_block_size; - mem_addr_t access_limit = offset + nbytes_remain; - if (access_limit > BSIZE) { - access_limit = BSIZE; - } - - size_t tx_bytes = access_limit - offset; - read_single_block(page, current_addr, tx_bytes, - &((unsigned char *)data)[dst_offset]); + while (nbytes_remain > 0) { + unsigned offset = current_addr & (BSIZE-1); + mem_addr_t page = current_addr >> m_log2_block_size; + mem_addr_t access_limit = offset + nbytes_remain; + if (access_limit > BSIZE) { + access_limit = BSIZE; + } + + size_t tx_bytes = access_limit - offset; + read_single_block(page, current_addr, tx_bytes, &((unsigned char*)data)[dst_offset]); - // advance pointers - dst_offset += tx_bytes; - current_addr += tx_bytes; - nbytes_remain -= tx_bytes; - } - assert(nbytes_remain == 0); - } + // advance pointers + dst_offset += tx_bytes; + current_addr += tx_bytes; + nbytes_remain -= tx_bytes; + } + assert(nbytes_remain == 0); + } } -template <unsigned BSIZE> -void memory_space_impl<BSIZE>::print(const char *format, FILE *fout) const { - typename map_t::const_iterator i_page; +template<unsigned BSIZE> void memory_space_impl<BSIZE>::print( const char *format, FILE *fout ) const +{ + typename map_t::const_iterator i_page; - for (i_page = m_data.begin(); i_page != m_data.end(); ++i_page) { - fprintf(fout, "%s %08x:", m_name.c_str(), i_page->first); - i_page->second.print(format, fout); - } + for ( i_page = m_data.begin(); i_page != m_data.end(); ++i_page) { + fprintf(fout, "%s %08x:", m_name.c_str(), i_page->first); + i_page->second.print(format, fout); + } } -template <unsigned BSIZE> -void memory_space_impl<BSIZE>::set_watch(addr_t addr, unsigned watchpoint) { - m_watchpoints[watchpoint] = addr; +template<unsigned BSIZE> void memory_space_impl<BSIZE>::set_watch( addr_t addr, unsigned watchpoint ) +{ + m_watchpoints[watchpoint]=addr; } template class memory_space_impl<32>; template class memory_space_impl<64>; template class memory_space_impl<8192>; -template class memory_space_impl<16 * 1024>; +template class memory_space_impl<16*1024>; -void g_print_memory_space(memory_space *mem, const char *format = "%08x", - FILE *fout = stdout) { - mem->print(format, fout); +void g_print_memory_space(memory_space *mem, const char *format = "%08x", FILE *fout = stdout) +{ + mem->print(format,fout); } #ifdef UNIT_TEST -int main(int argc, char *argv[]) { - int errors_found = 0; - memory_space *mem = new memory_space_impl<32>("test", 4); - // write address to [address] - for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 4) - mem->write(addr, 4, &addr, NULL, NULL); +int main(int argc, char *argv[] ) +{ + int errors_found=0; + memory_space *mem = new memory_space_impl<32>("test",4); + // write address to [address] + for( mem_addr_t addr=0; addr < 16*1024; addr+=4) + mem->write(addr,4,&addr,NULL,NULL); - for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 4) { - unsigned tmp = 0; - mem->read(addr, 4, &tmp); - if (tmp != addr) { - errors_found = 1; - printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, addr); - } - } + for( mem_addr_t addr=0; addr < 16*1024; addr+=4) { + unsigned tmp=0; + mem->read(addr,4,&tmp); + if( tmp != addr ) { + errors_found=1; + printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, addr ); + } + } - for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 1) { - unsigned char val = (addr + 128) % 256; - mem->write(addr, 1, &val, NULL, NULL); - } + for( mem_addr_t addr=0; addr < 16*1024; addr+=1) { + unsigned char val = (addr + 128) % 256; + mem->write(addr,1,&val,NULL,NULL); + } - for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 1) { - unsigned tmp = 0; - mem->read(addr, 1, &tmp); - unsigned char val = (addr + 128) % 256; - if (tmp != val) { - errors_found = 1; - printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, - (unsigned)val); - } - } + for( mem_addr_t addr=0; addr < 16*1024; addr+=1) { + unsigned tmp=0; + mem->read(addr,1,&tmp); + unsigned char val = (addr + 128) % 256; + if( tmp != val ) { + errors_found=1; + printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, (unsigned)val ); + } + } - if (errors_found) { - printf("SUMMARY: ERRORS FOUND\n"); - } else { - printf("SUMMARY: UNIT TEST PASSED\n"); - } + if( errors_found ) { + printf("SUMMARY: ERRORS FOUND\n"); + } else { + printf("SUMMARY: UNIT TEST PASSED\n"); + } } #endif diff --git a/src/cuda-sim/memory.h b/src/cuda-sim/memory.h index 31ad4df..ab588bc 100644 --- a/src/cuda-sim/memory.h +++ b/src/cuda-sim/memory.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -35,97 +33,101 @@ #include "../tr1_hash_map.h" #define mem_map tr1_hash_map #if tr1_hash_map_ismap == 1 -#define MEM_MAP_RESIZE(hash_size) + #define MEM_MAP_RESIZE(hash_size) #else -#define MEM_MAP_RESIZE(hash_size) (m_data.rehash(hash_size)) + #define MEM_MAP_RESIZE(hash_size) (m_data.rehash(hash_size)) #endif #include <assert.h> -#include <stdio.h> -#include <stdlib.h> #include <string.h> -#include <map> +#include <stdio.h> #include <string> +#include <map> +#include <stdlib.h> typedef address_type mem_addr_t; -#define MEM_BLOCK_SIZE (4 * 1024) +#define MEM_BLOCK_SIZE (4*1024) -template <unsigned BSIZE> -class mem_storage { - public: - mem_storage(const mem_storage &another) { - m_data = (unsigned char *)calloc(1, BSIZE); - memcpy(m_data, another.m_data, BSIZE); - } - mem_storage() { m_data = (unsigned char *)calloc(1, BSIZE); } - ~mem_storage() { free(m_data); } +template<unsigned BSIZE> class mem_storage { +public: + mem_storage( const mem_storage &another ) + { + m_data = (unsigned char*)calloc(1,BSIZE); + memcpy(m_data,another.m_data,BSIZE); + } + mem_storage() + { + m_data = (unsigned char*)calloc(1,BSIZE); + } + ~mem_storage() + { + free(m_data); + } - void write(unsigned offset, size_t length, const unsigned char *data) { - assert(offset + length <= BSIZE); - memcpy(m_data + offset, data, length); - } + void write( unsigned offset, size_t length, const unsigned char *data ) + { + assert( offset + length <= BSIZE ); + memcpy(m_data+offset,data,length); + } - void read(unsigned offset, size_t length, unsigned char *data) const { - assert(offset + length <= BSIZE); - memcpy(data, m_data + offset, length); - } + void read( unsigned offset, size_t length, unsigned char *data ) const + { + assert( offset + length <= BSIZE ); + memcpy(data,m_data+offset,length); + } - void print(const char *format, FILE *fout) const { - unsigned int *i_data = (unsigned int *)m_data; - for (int d = 0; d < (BSIZE / sizeof(unsigned int)); d++) { - if (d % 1 == 0) { - fprintf(fout, "\n"); + void print( const char *format, FILE *fout ) const + { + unsigned int *i_data = (unsigned int*)m_data; + for (int d = 0; d < (BSIZE / sizeof(unsigned int)); d++) { + if (d % 1 == 0) { + fprintf(fout, "\n"); + } + fprintf(fout, format, i_data[d]); + fprintf(fout, " "); } - fprintf(fout, format, i_data[d]); - fprintf(fout, " "); - } - fprintf(fout, "\n"); - fflush(fout); - } + fprintf(fout, "\n"); + fflush(fout); + } - private: - unsigned m_nbytes; - unsigned char *m_data; +private: + unsigned m_nbytes; + unsigned char *m_data; }; class ptx_thread_info; class ptx_instruction; -class memory_space { - public: - virtual ~memory_space() {} - virtual void write(mem_addr_t addr, size_t length, const void *data, - ptx_thread_info *thd, const ptx_instruction *pI) = 0; - virtual void write_only(mem_addr_t index, mem_addr_t offset, size_t length, - const void *data) = 0; - virtual void read(mem_addr_t addr, size_t length, void *data) const = 0; - virtual void print(const char *format, FILE *fout) const = 0; - virtual void set_watch(addr_t addr, unsigned watchpoint) = 0; +class memory_space +{ +public: + virtual ~memory_space() {} + virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ) = 0; + virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data ) = 0; + virtual void read( mem_addr_t addr, size_t length, void *data ) const = 0; + virtual void print( const char *format, FILE *fout ) const = 0; + virtual void set_watch( addr_t addr, unsigned watchpoint ) = 0; }; -template <unsigned BSIZE> -class memory_space_impl : public memory_space { - public: - memory_space_impl(std::string name, unsigned hash_size); - - virtual void write(mem_addr_t addr, size_t length, const void *data, - ptx_thread_info *thd, const ptx_instruction *pI); - virtual void write_only(mem_addr_t index, mem_addr_t offset, size_t length, - const void *data); - virtual void read(mem_addr_t addr, size_t length, void *data) const; - virtual void print(const char *format, FILE *fout) const; +template<unsigned BSIZE> class memory_space_impl : public memory_space { +public: + memory_space_impl( std::string name, unsigned hash_size ); - virtual void set_watch(addr_t addr, unsigned watchpoint); + virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ); + virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data); + virtual void read( mem_addr_t addr, size_t length, void *data ) const; + virtual void print( const char *format, FILE *fout ) const; + + virtual void set_watch( addr_t addr, unsigned watchpoint ); - private: - void read_single_block(mem_addr_t blk_idx, mem_addr_t addr, size_t length, - void *data) const; - std::string m_name; - unsigned m_log2_block_size; - typedef mem_map<mem_addr_t, mem_storage<BSIZE> > map_t; - map_t m_data; - std::map<unsigned, mem_addr_t> m_watchpoints; +private: + void read_single_block( mem_addr_t blk_idx, mem_addr_t addr, size_t length, void *data) const; + std::string m_name; + unsigned m_log2_block_size; + typedef mem_map<mem_addr_t,mem_storage<BSIZE> > map_t; + map_t m_data; + std::map<unsigned,mem_addr_t> m_watchpoints; }; #endif diff --git a/src/cuda-sim/opcodes.h b/src/cuda-sim/opcodes.h index 479b1ca..86d3b99 100644 --- a/src/cuda-sim/opcodes.h +++ b/src/cuda-sim/opcodes.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -31,48 +29,49 @@ #define opcodes_h_included enum opcode_t { -#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) OP, -#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) OP, +#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP, +#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP, #include "opcodes.def" - NUM_OPCODES + NUM_OPCODES #undef OP_DEF #undef OP_W_DEF }; enum special_regs { - CLOCK_REG, - HALFCLOCK_ID, - CLOCK64_REG, - CTAID_REG, - ENVREG_REG, - GRIDID_REG, - LANEID_REG, - LANEMASK_EQ_REG, - LANEMASK_LE_REG, - LANEMASK_LT_REG, - LANEMASK_GE_REG, - LANEMASK_GT_REG, - NCTAID_REG, - NTID_REG, - NSMID_REG, - NWARPID_REG, - PM_REG, - SMID_REG, - TID_REG, - WARPID_REG, - WARPSZ_REG + CLOCK_REG, + HALFCLOCK_ID, + CLOCK64_REG, + CTAID_REG, + ENVREG_REG, + GRIDID_REG, + LANEID_REG, + LANEMASK_EQ_REG, + LANEMASK_LE_REG, + LANEMASK_LT_REG, + LANEMASK_GE_REG, + LANEMASK_GT_REG, + NCTAID_REG, + NTID_REG, + NSMID_REG, + NWARPID_REG, + PM_REG, + SMID_REG, + TID_REG, + WARPID_REG, + WARPSZ_REG }; -enum wmma_type { - LOAD_A, - LOAD_B, - LOAD_C, - STORE_D, - MMA, - ROW, - COL, - M16N16K16, - M32N8K16, - M8N32K16 +enum wmma_type{ + LOAD_A, + LOAD_B, + LOAD_C, + STORE_D, + MMA, + ROW, + COL, + M16N16K16, + M32N8K16, + M8N32K16 + }; #endif diff --git a/src/cuda-sim/ptx-stats.cc b/src/cuda-sim/ptx-stats.cc index 2f16555..22517df 100644 --- a/src/cuda-sim/ptx-stats.cc +++ b/src/cuda-sim/ptx-stats.cc @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -27,266 +25,248 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "ptx_ir.h" +#include "ptx_sim.h" #include "ptx-stats.h" +#include "../option_parser.h" #include <stdio.h> #include <map> -#include "../../libcuda/gpgpu_context.h" -#include "../option_parser.h" #include "../tr1_hash_map.h" -#include "ptx_ir.h" -#include "ptx_sim.h" +#include "../../libcuda/gpgpu_context.h" -void ptx_stats::ptx_file_line_stats_options(option_parser_t opp) { - option_parser_register( - opp, "-enable_ptx_file_line_stats", OPT_BOOL, &enable_ptx_file_line_stats, - "Turn on PTX source line statistic profiling. (1 = On)", "1"); - option_parser_register( - opp, "-ptx_line_stats_filename", OPT_CSTR, &ptx_line_stats_filename, - "Output file for PTX source line statistics.", "gpgpu_inst_stats.txt"); +void ptx_stats::ptx_file_line_stats_options(option_parser_t opp) +{ + option_parser_register(opp, "-enable_ptx_file_line_stats", OPT_BOOL, + &enable_ptx_file_line_stats, + "Turn on PTX source line statistic profiling. (1 = On)", "1"); + option_parser_register(opp, "-ptx_line_stats_filename", OPT_CSTR, + &ptx_line_stats_filename, + "Output file for PTX source line statistics.", "gpgpu_inst_stats.txt"); } // implementations // defining a PTX source line = filename + line number -class ptx_file_line { - public: - ptx_file_line(const char *s, int l) { - if (s == NULL) - st = "NULL_NAME"; - else - st = s; - line = l; - } +class ptx_file_line +{ +public: + ptx_file_line(const char* s, int l) { + if( s == NULL ) + st = "NULL_NAME"; + else + st = s; + line = l; + } - bool operator<(const ptx_file_line &other) const { - if (st == other.st) { - if (line < other.line) - return true; - else - return false; - } else { - return st < other.st; + bool operator<(const ptx_file_line &other) const { + if( st == other.st ) { + if( line < other.line ) + return true; + else + return false; + } else { + return st < other.st; + } } - } - bool operator==(const ptx_file_line &other) const { - return (line == other.line) && (st == other.st); - } + bool operator==(const ptx_file_line &other) const { + return (line==other.line) && (st==other.st); + } - std::string st; - unsigned line; + std::string st; + unsigned line; }; // holds all statistics collected for a singe PTX source line -class ptx_file_line_stats { - public: - ptx_file_line_stats() - : exec_count(0), - latency(0), - dram_traffic(0), - smem_n_way_bank_conflict_total(0), - smem_warp_count(0), - gmem_n_access_total(0), - gmem_warp_count(0), - exposed_latency(0), - warp_divergence(0) {} - - unsigned long exec_count; - unsigned long long latency; - unsigned long long dram_traffic; - unsigned long long smem_n_way_bank_conflict_total; // total number of banks - // accessed by this - // instruction - unsigned long smem_warp_count; // number of warps accessing shared memory - unsigned long long gmem_n_access_total; // number of uncoalesced access in - // total from this instruction - unsigned long - gmem_warp_count; // number of warps causing these uncoalesced access - unsigned long long exposed_latency; // latency exposed as pipeline bubbles - // (attributed to this instruction) - unsigned long long - warp_divergence; // number of warp divergence occured at this instruction +class ptx_file_line_stats +{ +public: + ptx_file_line_stats() + : exec_count(0), latency(0), dram_traffic(0), + smem_n_way_bank_conflict_total(0), smem_warp_count(0), + gmem_n_access_total(0), gmem_warp_count(0), exposed_latency(0), + warp_divergence(0) + { } + + unsigned long exec_count; + unsigned long long latency; + unsigned long long dram_traffic; + unsigned long long smem_n_way_bank_conflict_total; // total number of banks accessed by this instruction + unsigned long smem_warp_count; // number of warps accessing shared memory + unsigned long long gmem_n_access_total; // number of uncoalesced access in total from this instruction + unsigned long gmem_warp_count; // number of warps causing these uncoalesced access + unsigned long long exposed_latency; // latency exposed as pipeline bubbles (attributed to this instruction) + unsigned long long warp_divergence; // number of warp divergence occured at this instruction }; #if (tr1_hash_map_ismap == 1) -typedef tr1_hash_map<ptx_file_line, ptx_file_line_stats> - ptx_file_line_stats_map_t; +typedef tr1_hash_map<ptx_file_line, ptx_file_line_stats> ptx_file_line_stats_map_t; #else -struct hash_ptx_file_line { - std::size_t operator()(const ptx_file_line &pfline) const { - std::hash<unsigned> hash_line; - return hash_line(pfline.line); - } +struct hash_ptx_file_line +{ + std::size_t operator()(const ptx_file_line & pfline) const { + std::hash<unsigned> hash_line; + return hash_line(pfline.line); + } }; -typedef tr1_hash_map<ptx_file_line, ptx_file_line_stats, hash_ptx_file_line> - ptx_file_line_stats_map_t; +typedef tr1_hash_map<ptx_file_line, ptx_file_line_stats, hash_ptx_file_line> ptx_file_line_stats_map_t; #endif static ptx_file_line_stats_map_t ptx_file_line_stats_tracker; // output statistics to a file -void ptx_stats::ptx_file_line_stats_write_file() { - // check if stat collection is turned on - if (enable_ptx_file_line_stats == 0) return; +void ptx_stats::ptx_file_line_stats_write_file() +{ + // check if stat collection is turned on + if (enable_ptx_file_line_stats == 0) return; - ptx_file_line_stats_map_t::iterator it; - FILE *pfile; + ptx_file_line_stats_map_t::iterator it; + FILE * pfile; - pfile = fopen(ptx_line_stats_filename, "w"); - fprintf(pfile, - "kernel line : count latency dram_traffic smem_bk_conflicts " - "smem_warp gmem_access_generated gmem_warp exposed_latency " - "warp_divergence\n"); - for (it = ptx_file_line_stats_tracker.begin(); - it != ptx_file_line_stats_tracker.end(); it++) { - fprintf(pfile, "%s %i : ", it->first.st.c_str(), it->first.line); - fprintf(pfile, "%lu ", it->second.exec_count); - fprintf(pfile, "%llu ", it->second.latency); - fprintf(pfile, "%llu ", it->second.dram_traffic); - fprintf(pfile, "%llu ", it->second.smem_n_way_bank_conflict_total); - fprintf(pfile, "%lu ", it->second.smem_warp_count); - fprintf(pfile, "%llu ", it->second.gmem_n_access_total); - fprintf(pfile, "%lu ", it->second.gmem_warp_count); - fprintf(pfile, "%llu ", it->second.exposed_latency); - fprintf(pfile, "%llu ", it->second.warp_divergence); - fprintf(pfile, "\n"); - } - fflush(pfile); - fclose(pfile); + pfile = fopen(ptx_line_stats_filename, "w"); + fprintf(pfile,"kernel line : count latency dram_traffic smem_bk_conflicts smem_warp gmem_access_generated gmem_warp exposed_latency warp_divergence\n"); + for( it=ptx_file_line_stats_tracker.begin(); it != ptx_file_line_stats_tracker.end(); it++ ) { + fprintf(pfile, "%s %i : ", it->first.st.c_str(), it->first.line); + fprintf(pfile, "%lu ", it->second.exec_count); + fprintf(pfile, "%llu ", it->second.latency); + fprintf(pfile, "%llu ", it->second.dram_traffic); + fprintf(pfile, "%llu ", it->second.smem_n_way_bank_conflict_total); + fprintf(pfile, "%lu ", it->second.smem_warp_count); + fprintf(pfile, "%llu ", it->second.gmem_n_access_total); + fprintf(pfile, "%lu ", it->second.gmem_warp_count); + fprintf(pfile, "%llu ", it->second.exposed_latency); + fprintf(pfile, "%llu ", it->second.warp_divergence); + fprintf(pfile, "\n"); + } + fflush(pfile); + fclose(pfile); } // attribute one more execution count to this ptx instruction // counting the number of threads (not warps) executing this instruction -void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn) { - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), - pInsn->source_line())] - .exec_count += 1; +void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn) +{ + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].exec_count += 1; } // attribute pipeline latency to this ptx instruction (specified by the pc) -// pipeline latency is the number of cycles a warp with this instruction spent -// in the pipeline -void ptx_stats::ptx_file_line_stats_add_latency(unsigned pc, unsigned latency) { - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), - pInsn->source_line())] - .latency += latency; +// pipeline latency is the number of cycles a warp with this instruction spent in the pipeline +void ptx_stats::ptx_file_line_stats_add_latency(unsigned pc, unsigned latency) +{ + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].latency += latency; } // attribute dram traffic to this ptx instruction (specified by the pc) -// dram traffic is counted in number of requests -void ptx_stats::ptx_file_line_stats_add_dram_traffic(unsigned pc, - unsigned dram_traffic) { - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), - pInsn->source_line())] - .dram_traffic += dram_traffic; +// dram traffic is counted in number of requests +void ptx_stats::ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic) +{ + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].dram_traffic += dram_traffic; } // attribute the number of shared memory access cycles to a ptx instruction -// counts both the number of warps doing shared memory access and the number of -// cycles involved -void ptx_stats::ptx_file_line_stats_add_smem_bank_conflict( - unsigned pc, unsigned n_way_bkconflict) { - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( - pInsn->source_file(), pInsn->source_line())]; - line_stats.smem_n_way_bank_conflict_total += n_way_bkconflict; - line_stats.smem_warp_count += 1; +// counts both the number of warps doing shared memory access and the number of cycles involved +void ptx_stats::ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, unsigned n_way_bkconflict) +{ + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; + line_stats.smem_n_way_bank_conflict_total += n_way_bkconflict; + line_stats.smem_warp_count += 1; } -// attribute a non-coalesced mem access to a ptx instruction -// counts both the number of warps causing this and the number of memory -// requests generated -void ptx_stats::ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, - unsigned n_access) { - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( - pInsn->source_file(), pInsn->source_line())]; - line_stats.gmem_n_access_total += n_access; - line_stats.gmem_warp_count += 1; +// attribute a non-coalesced mem access to a ptx instruction +// counts both the number of warps causing this and the number of memory requests generated +void ptx_stats::ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access) +{ + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; + line_stats.gmem_n_access_total += n_access; + line_stats.gmem_warp_count += 1; } -// a class that tracks the inflight memory instructions of a shader core -// and attributes exposed latency to those instructions when signaled to do so -class ptx_inflight_memory_insn_tracker { - public: - typedef std::map<const ptx_instruction *, int> insn_count_map; +// a class that tracks the inflight memory instructions of a shader core +// and attributes exposed latency to those instructions when signaled to do so +class ptx_inflight_memory_insn_tracker +{ +public: + typedef std::map<const ptx_instruction *, int> insn_count_map; - void add_count(const ptx_instruction *pInsn, int count = 1) { - ptx_inflight_memory_insns[pInsn] += count; - } + void add_count(const ptx_instruction * pInsn, int count = 1) + { + ptx_inflight_memory_insns[pInsn] += count; + } - void sub_count(const ptx_instruction *pInsn, int count = 1) { - insn_count_map::iterator i_insncount; - i_insncount = ptx_inflight_memory_insns.find(pInsn); + void sub_count(const ptx_instruction * pInsn, int count = 1) + { + insn_count_map::iterator i_insncount; + i_insncount = ptx_inflight_memory_insns.find(pInsn); - assert(i_insncount != ptx_inflight_memory_insns.end()); + assert(i_insncount != ptx_inflight_memory_insns.end()); - i_insncount->second -= count; + i_insncount->second -= count; - if (i_insncount->second <= 0) { - ptx_inflight_memory_insns.erase(i_insncount); + if (i_insncount->second <= 0) { + ptx_inflight_memory_insns.erase(i_insncount); + } } - } - void attribute_exposed_latency(int count = 1) { - insn_count_map &exlat_insnmap = ptx_inflight_memory_insns; - insn_count_map::const_iterator i_exlatinsn; + void attribute_exposed_latency(int count = 1) + { + insn_count_map &exlat_insnmap = ptx_inflight_memory_insns; + insn_count_map::const_iterator i_exlatinsn; - i_exlatinsn = exlat_insnmap.begin(); - for (; i_exlatinsn != exlat_insnmap.end(); ++i_exlatinsn) { - const ptx_instruction *pInsn = i_exlatinsn->first; - ptx_file_line_stats &line_stats = - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), - pInsn->source_line())]; - line_stats.exposed_latency += count; + i_exlatinsn = exlat_insnmap.begin(); + for (; i_exlatinsn != exlat_insnmap.end(); ++i_exlatinsn) { + const ptx_instruction *pInsn = i_exlatinsn->first; + ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; + line_stats.exposed_latency += count; + } } - } - insn_count_map ptx_inflight_memory_insns; + insn_count_map ptx_inflight_memory_insns; }; static ptx_inflight_memory_insn_tracker *inflight_mem_tracker = NULL; -void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores) { - inflight_mem_tracker = new ptx_inflight_memory_insn_tracker[n_shader_cores]; +void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores) +{ + inflight_mem_tracker = new ptx_inflight_memory_insn_tracker[n_shader_cores]; } // add an inflight memory instruction -void ptx_stats::ptx_file_line_stats_add_inflight_memory_insn(int sc_id, - unsigned pc) { - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); +void ptx_stats::ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc) +{ + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - inflight_mem_tracker[sc_id].add_count(pInsn); + inflight_mem_tracker[sc_id].add_count(pInsn); } // remove an inflight memory instruction -void ptx_stats::ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, - unsigned pc) { - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); +void ptx_stats::ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc) +{ + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - inflight_mem_tracker[sc_id].sub_count(pInsn); + inflight_mem_tracker[sc_id].sub_count(pInsn); } -// attribute an empty cycle in the pipeline (exposed latency) to the ptx memory -// instructions in flight -void ptx_file_line_stats_commit_exposed_latency(int sc_id, - int exposed_latency) { - assert(exposed_latency > 0); - inflight_mem_tracker[sc_id].attribute_exposed_latency(exposed_latency); +// attribute an empty cycle in the pipeline (exposed latency) to the ptx memory instructions in flight +void ptx_file_line_stats_commit_exposed_latency(int sc_id, int exposed_latency) +{ + assert(exposed_latency > 0); + inflight_mem_tracker[sc_id].attribute_exposed_latency(exposed_latency); } // attribute the number of warp divergence to a ptx instruction -void ptx_stats::ptx_file_line_stats_add_warp_divergence( - unsigned pc, unsigned n_way_divergence) { - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( - pInsn->source_file(), pInsn->source_line())]; - line_stats.warp_divergence += n_way_divergence; +void ptx_stats::ptx_file_line_stats_add_warp_divergence(unsigned pc, unsigned n_way_divergence) +{ + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; + line_stats.warp_divergence += n_way_divergence; } + diff --git a/src/cuda-sim/ptx-stats.h b/src/cuda-sim/ptx-stats.h index 7745eba..246b4ce 100644 --- a/src/cuda-sim/ptx-stats.h +++ b/src/cuda-sim/ptx-stats.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -27,14 +25,15 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#pragma once +#pragma once #include "../option_parser.h" + #ifdef __cplusplus // stat collection interface to cuda-sim class ptx_instruction; -void ptx_file_line_stats_add_exec_count(const ptx_instruction* pInsn); +void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn); #endif // stat collection interface to gpgpu-sim @@ -42,29 +41,28 @@ void ptx_file_line_stats_add_exec_count(const ptx_instruction* pInsn); void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores); void ptx_file_line_stats_commit_exposed_latency(int sc_id, int exposed_latency); + class gpgpu_context; class ptx_stats { - public: - ptx_stats(gpgpu_context* ctx) { - ptx_line_stats_filename = NULL; - gpgpu_ctx = ctx; - } - char* ptx_line_stats_filename; - bool enable_ptx_file_line_stats; - gpgpu_context* gpgpu_ctx; - // set options - void ptx_file_line_stats_options(option_parser_t opp); + public: + ptx_stats(gpgpu_context* ctx) { + ptx_line_stats_filename = NULL; + gpgpu_ctx = ctx; + } + char * ptx_line_stats_filename; + bool enable_ptx_file_line_stats; + gpgpu_context* gpgpu_ctx; + // set options + void ptx_file_line_stats_options(option_parser_t opp); - // output stats to a file - void ptx_file_line_stats_write_file(); - // stat collection interface to gpgpu-sim - void ptx_file_line_stats_add_latency(unsigned pc, unsigned latency); - void ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic); - void ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, - unsigned n_way_bkconflict); - void ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access); - void ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc); - void ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc); - void ptx_file_line_stats_add_warp_divergence(unsigned pc, - unsigned n_way_divergence); + // output stats to a file + void ptx_file_line_stats_write_file(); + // stat collection interface to gpgpu-sim + void ptx_file_line_stats_add_latency(unsigned pc, unsigned latency); + void ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic); + void ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, unsigned n_way_bkconflict); + void ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access); + void ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc); + void ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc); + void ptx_file_line_stats_add_warp_divergence(unsigned pc, unsigned n_way_divergence); }; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index 55afb51..d8943d2 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -1,5 +1,5 @@ // Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda, Wilson W.L. Fung, -// George L. Yuan +// George L. Yuan // The University of British Columbia // All rights reserved. // @@ -8,16 +8,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -28,1244 +26,1152 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#include "ptx_ir.h" #include "ptx_parser.h" -typedef void *yyscan_t; -#include <assert.h> +#include "ptx_ir.h" +typedef void * yyscan_t; +#include "ptx.tab.h" +#include "opcodes.h" #include <stdio.h> #include <stdlib.h> -#include <algorithm> #include <list> +#include <assert.h> +#include <algorithm> #include "assert.h" -#include "opcodes.h" -#include "ptx.tab.h" -#include "../../libcuda/gpgpu_context.h" #include "cuda-sim.h" +#include "../../libcuda/gpgpu_context.h" #define STR_SIZE 1024 -const ptx_instruction *gpgpu_context::pc_to_instruction(unsigned pc) { - if (pc < s_g_pc_to_insn.size()) - return s_g_pc_to_insn[pc]; - else - return NULL; +const ptx_instruction* gpgpu_context::pc_to_instruction(unsigned pc) +{ + if( pc < s_g_pc_to_insn.size() ) + return s_g_pc_to_insn[pc]; + else + return NULL; } -unsigned symbol::get_uid() { - unsigned result = (gpgpu_ctx->symbol_sm_next_uid)++; - return result; +unsigned symbol::get_uid() +{ + unsigned result = (gpgpu_ctx->symbol_sm_next_uid)++; + return result; } -void symbol::add_initializer(const std::list<operand_info> &init) { - m_initializer = init; +void symbol::add_initializer( const std::list<operand_info> &init ) +{ + m_initializer = init; } -void symbol::print_info(FILE *fp) const { - fprintf(fp, "uid:%u, decl:%s, type:%p, ", m_uid, m_decl_location.c_str(), - m_type); - if (m_address_valid) fprintf(fp, "<address valid>, "); - if (m_is_label) fprintf(fp, " is_label "); - if (m_is_shared) fprintf(fp, " is_shared "); - if (m_is_const) fprintf(fp, " is_const "); - if (m_is_global) fprintf(fp, " is_global "); - if (m_is_local) fprintf(fp, " is_local "); - if (m_is_tex) fprintf(fp, " is_tex "); - if (m_is_func_addr) fprintf(fp, " is_func_addr "); - if (m_function) fprintf(fp, " %p ", m_function); +void symbol::print_info(FILE *fp) const +{ + fprintf(fp,"uid:%u, decl:%s, type:%p, ", m_uid, m_decl_location.c_str(), m_type ); + if( m_address_valid ) + fprintf(fp,"<address valid>, "); + if( m_is_label ) + fprintf(fp," is_label "); + if( m_is_shared ) + fprintf(fp," is_shared "); + if( m_is_const ) + fprintf(fp," is_const "); + if( m_is_global ) + fprintf(fp," is_global "); + if( m_is_local ) + fprintf(fp," is_local "); + if( m_is_tex ) + fprintf(fp," is_tex "); + if( m_is_func_addr ) + fprintf(fp," is_func_addr "); + if( m_function ) + fprintf(fp," %p ", m_function ); } -symbol_table::symbol_table() { assert(0); } +symbol_table::symbol_table() +{ + assert(0); +} -symbol_table::symbol_table(const char *scope_name, unsigned entry_point, - symbol_table *parent, gpgpu_context *ctx) { - gpgpu_ctx = ctx; - m_scope_name = std::string(scope_name); - m_reg_allocator = 0; - m_shared_next = 0; - m_const_next = 0; - m_global_next = 0x100; - m_local_next = 0; - m_tex_next = 0; +symbol_table::symbol_table( const char *scope_name, unsigned entry_point, symbol_table *parent, gpgpu_context* ctx ) +{ + gpgpu_ctx = ctx; + m_scope_name = std::string(scope_name); + m_reg_allocator=0; + m_shared_next = 0; + m_const_next = 0; + m_global_next = 0x100; + m_local_next = 0; + m_tex_next = 0; - // Jin: handle instruction group for cdp - m_inst_group_id = 0; + //Jin: handle instruction group for cdp + m_inst_group_id = 0; - m_parent = parent; - if (m_parent) { - m_shared_next = m_parent->m_shared_next; - m_global_next = m_parent->m_global_next; - } + m_parent = parent; + if ( m_parent ) { + m_shared_next = m_parent->m_shared_next; + m_global_next = m_parent->m_global_next; + } } -void symbol_table::set_name(const char *name) { - m_scope_name = std::string(name); +void symbol_table::set_name( const char *name ) +{ + m_scope_name = std::string(name); } -const ptx_version &symbol_table::get_ptx_version() const { - if (m_parent == NULL) - return m_ptx_version; - else - return m_parent->get_ptx_version(); +const ptx_version &symbol_table::get_ptx_version() const +{ + if( m_parent == NULL ) return m_ptx_version; + else return m_parent->get_ptx_version(); } -unsigned symbol_table::get_sm_target() const { - if (m_parent == NULL) - return m_ptx_version.target(); - else - return m_parent->get_sm_target(); +unsigned symbol_table::get_sm_target() const +{ + if( m_parent == NULL ) + return m_ptx_version.target(); + else return m_parent->get_sm_target(); } -void symbol_table::set_ptx_version(float ver, unsigned ext) { - m_ptx_version = ptx_version(ver, ext); +void symbol_table::set_ptx_version( float ver, unsigned ext ) +{ + m_ptx_version = ptx_version(ver,ext); } -void symbol_table::set_sm_target(const char *target, const char *ext, - const char *ext2) { - m_ptx_version.set_target(target, ext, ext2); +void symbol_table::set_sm_target( const char *target, const char *ext, const char *ext2 ) +{ + m_ptx_version.set_target(target,ext,ext2); } -symbol *symbol_table::lookup(const char *identifier) { - std::string key(identifier); - std::map<std::string, symbol *>::iterator i = m_symbols.find(key); - if (i != m_symbols.end()) { - return i->second; - } - if (m_parent) { - return m_parent->lookup(identifier); - } - return NULL; +symbol *symbol_table::lookup( const char *identifier ) +{ + std::string key(identifier); + std::map<std::string, symbol *>::iterator i = m_symbols.find(key); + if ( i != m_symbols.end() ) { + return i->second; + } + if ( m_parent ) { + return m_parent->lookup(identifier); + } + return NULL; } -symbol *symbol_table::add_variable(const char *identifier, - const type_info *type, unsigned size, - const char *filename, unsigned line) { - char buf[1024]; - std::string key(identifier); - assert(m_symbols.find(key) == m_symbols.end()); - snprintf(buf, 1024, "%s:%u", filename, line); - symbol *s = new symbol(identifier, type, buf, size, gpgpu_ctx); - m_symbols[key] = s; +symbol *symbol_table::add_variable( const char *identifier, const type_info *type, unsigned size, const char *filename, unsigned line ) +{ + char buf[1024]; + std::string key(identifier); + assert( m_symbols.find(key) == m_symbols.end() ); + snprintf(buf,1024,"%s:%u",filename,line); + symbol *s = new symbol(identifier,type,buf,size,gpgpu_ctx); + m_symbols[ key ] = s; - if (type != NULL && type->get_key().is_global()) { - m_globals.push_back(s); - } - if (type != NULL && type->get_key().is_const()) { - m_consts.push_back(s); - } + if ( type != NULL && type->get_key().is_global() ) { + m_globals.push_back(s); + } + if ( type != NULL && type->get_key().is_const() ) { + m_consts.push_back(s); + } - return s; + return s; } -void symbol_table::add_function(function_info *func, const char *filename, - unsigned linenumber) { - std::map<std::string, symbol *>::iterator i = - m_symbols.find(func->get_name()); - if (i != m_symbols.end()) return; - char buf[1024]; - snprintf(buf, 1024, "%s:%u", filename, linenumber); - type_info *type = add_type(func); - symbol *s = new symbol(func->get_name().c_str(), type, buf, 0, gpgpu_ctx); - s->set_function(func); - m_symbols[func->get_name()] = s; +void symbol_table::add_function( function_info *func, const char *filename, unsigned linenumber ) +{ + std::map<std::string, symbol *>::iterator i = m_symbols.find( func->get_name() ); + if( i != m_symbols.end() ) + return; + char buf[1024]; + snprintf(buf,1024,"%s:%u",filename,linenumber); + type_info *type = add_type( func ); + symbol *s = new symbol(func->get_name().c_str(),type,buf,0,gpgpu_ctx); + s->set_function(func); + m_symbols[ func->get_name() ] = s; } -// Jin: handle instruction group for cdp -symbol_table *symbol_table::start_inst_group() { - char inst_group_name[4096]; - snprintf(inst_group_name, 4096, "%s_inst_group_%u", m_scope_name.c_str(), - m_inst_group_id); - - // previous added - assert(m_inst_group_symtab.find(std::string(inst_group_name)) == - m_inst_group_symtab.end()); - symbol_table *sym_table = - new symbol_table(inst_group_name, 3 /*inst group*/, this, gpgpu_ctx); +//Jin: handle instruction group for cdp +symbol_table* symbol_table::start_inst_group() { + char inst_group_name[4096]; + snprintf(inst_group_name, 4096, "%s_inst_group_%u", m_scope_name.c_str(), m_inst_group_id); - sym_table->m_global_next = m_global_next; - sym_table->m_shared_next = m_shared_next; - sym_table->m_local_next = m_local_next; - sym_table->m_reg_allocator = m_reg_allocator; - sym_table->m_tex_next = m_tex_next; - sym_table->m_const_next = m_const_next; + //previous added + assert(m_inst_group_symtab.find(std::string(inst_group_name)) == m_inst_group_symtab.end()); + symbol_table *sym_table = new symbol_table(inst_group_name, 3/*inst group*/, this, gpgpu_ctx ); + + sym_table->m_global_next = m_global_next; + sym_table->m_shared_next = m_shared_next; + sym_table->m_local_next = m_local_next; + sym_table->m_reg_allocator = m_reg_allocator; + sym_table->m_tex_next = m_tex_next; + sym_table->m_const_next = m_const_next; - m_inst_group_symtab[std::string(inst_group_name)] = sym_table; + m_inst_group_symtab[std::string(inst_group_name)] = sym_table; - return sym_table; + return sym_table; } -symbol_table *symbol_table::end_inst_group() { - symbol_table *sym_table = m_parent; +symbol_table * symbol_table::end_inst_group() { + symbol_table * sym_table = m_parent; + + sym_table->m_global_next = m_global_next; + sym_table->m_shared_next = m_shared_next; + sym_table->m_local_next = m_local_next; + sym_table->m_reg_allocator = m_reg_allocator; + sym_table->m_tex_next = m_tex_next; + sym_table->m_const_next = m_const_next; + sym_table->m_inst_group_id++; - sym_table->m_global_next = m_global_next; - sym_table->m_shared_next = m_shared_next; - sym_table->m_local_next = m_local_next; - sym_table->m_reg_allocator = m_reg_allocator; - sym_table->m_tex_next = m_tex_next; - sym_table->m_const_next = m_const_next; - sym_table->m_inst_group_id++; - - return sym_table; + return sym_table; } -void register_ptx_function(const char *name, - function_info *impl); // either libcuda or libopencl - -bool symbol_table::add_function_decl(const char *name, int entry_point, - function_info **func_info, - symbol_table **sym_table) { - std::string key = std::string(name); - bool prior_decl = false; - if (m_function_info_lookup.find(key) != m_function_info_lookup.end()) { - *func_info = m_function_info_lookup[key]; - prior_decl = true; - } else { - *func_info = new function_info(entry_point, gpgpu_ctx); - (*func_info)->set_name(name); - (*func_info)->set_maxnt_id(0); - m_function_info_lookup[key] = *func_info; - } +void register_ptx_function( const char *name, function_info *impl ); // either libcuda or libopencl - if (m_function_symtab_lookup.find(key) != m_function_symtab_lookup.end()) { - assert(prior_decl); - *sym_table = m_function_symtab_lookup[key]; - } else { - assert(!prior_decl); - *sym_table = new symbol_table("", entry_point, this, gpgpu_ctx); - - // Initial setup code to support a register represented as "_". - // This register is used when an instruction operand is - // not read or written. However, the parser must recognize it - // as a legitimate register but we do not want to pass - // it to the micro-architectural register to the performance simulator. - // For this purpose we add a symbol to the symbol table but - // mark it as a non_arch_reg so it does not effect the performance sim. - type_info_key null_key(reg_space, 0, 0, 0, 0, 0); - null_key.set_is_non_arch_reg(); - // First param is null - which is bad. - // However, the first parameter is actually unread in the constructor... - // TODO - remove the symbol_table* from type_info - type_info *null_type_info = new type_info(NULL, null_key); - symbol *null_reg = - (*sym_table)->add_variable("_", null_type_info, 0, "", 0); - null_reg->set_regno(0, 0); +bool symbol_table::add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **sym_table ) +{ + std::string key = std::string(name); + bool prior_decl = false; + if( m_function_info_lookup.find(key) != m_function_info_lookup.end() ) { + *func_info = m_function_info_lookup[key]; + prior_decl = true; + } else { + *func_info = new function_info(entry_point, gpgpu_ctx); + (*func_info)->set_name(name); + (*func_info)->set_maxnt_id(0); + m_function_info_lookup[key] = *func_info; + } - (*sym_table)->set_name(name); - (*func_info)->set_symtab(*sym_table); - m_function_symtab_lookup[key] = *sym_table; - assert((*func_info)->get_symtab() == *sym_table); - register_ptx_function(name, *func_info); - } - return prior_decl; + if( m_function_symtab_lookup.find(key) != m_function_symtab_lookup.end() ) { + assert( prior_decl ); + *sym_table = m_function_symtab_lookup[key]; + } else { + assert( !prior_decl ); + *sym_table = new symbol_table( "", entry_point, this, gpgpu_ctx ); + + // Initial setup code to support a register represented as "_". + // This register is used when an instruction operand is + // not read or written. However, the parser must recognize it + // as a legitimate register but we do not want to pass + // it to the micro-architectural register to the performance simulator. + // For this purpose we add a symbol to the symbol table but + // mark it as a non_arch_reg so it does not effect the performance sim. + type_info_key null_key( reg_space, 0, 0, 0, 0, 0 ); + null_key.set_is_non_arch_reg(); + // First param is null - which is bad. + // However, the first parameter is actually unread in the constructor... + // TODO - remove the symbol_table* from type_info + type_info* null_type_info = new type_info( NULL, null_key ); + symbol *null_reg = (*sym_table)->add_variable( "_", null_type_info, 0, "", 0 ); + null_reg->set_regno(0, 0); + + (*sym_table)->set_name(name); + (*func_info)->set_symtab(*sym_table); + m_function_symtab_lookup[key] = *sym_table; + assert( (*func_info)->get_symtab() == *sym_table ); + register_ptx_function(name,*func_info); + } + return prior_decl; } -function_info *symbol_table::lookup_function(std::string name) { - std::string key = std::string(name); - std::map<std::string, function_info *>::iterator it = - m_function_info_lookup.find(key); - assert(it != m_function_info_lookup.end()); - return it->second; +function_info *symbol_table::lookup_function( std::string name ) +{ + std::string key = std::string(name); + std::map<std::string,function_info*>::iterator it = m_function_info_lookup.find(key); + assert ( it != m_function_info_lookup.end() ); + return it->second; } -type_info *symbol_table::add_type(memory_space_t space_spec, - int scalar_type_spec, int vector_spec, - int alignment_spec, int extern_spec) { - if (space_spec == param_space_unclassified) space_spec = param_space_local; - type_info_key t(space_spec, scalar_type_spec, vector_spec, alignment_spec, - extern_spec, 0); - type_info *pt; - pt = new type_info(this, t); - return pt; +type_info *symbol_table::add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ) +{ + if( space_spec == param_space_unclassified ) + space_spec = param_space_local; + type_info_key t(space_spec,scalar_type_spec,vector_spec,alignment_spec,extern_spec,0); + type_info *pt; + pt = new type_info(this,t); + return pt; } -type_info *symbol_table::add_type(function_info *func) { - type_info_key t; - type_info *pt; - t.set_is_func(); - pt = new type_info(this, t); - return pt; +type_info *symbol_table::add_type( function_info *func ) +{ + type_info_key t; + type_info *pt; + t.set_is_func(); + pt = new type_info(this,t); + return pt; } -type_info *symbol_table::get_array_type(type_info *base_type, - unsigned array_dim) { - type_info_key t = base_type->get_key(); - t.set_array_dim(array_dim); - type_info *pt = new type_info(this, t); - // Where else is m_types being used? As of now, I dont find any use of it and - // causing seg fault. So disabling m_types. - // TODO: find where m_types can be used in future and solve the seg fault. - // pt = m_types[t] = new type_info(this,t); - return pt; +type_info *symbol_table::get_array_type( type_info *base_type, unsigned array_dim ) +{ + type_info_key t = base_type->get_key(); + t.set_array_dim(array_dim); + type_info *pt = new type_info(this,t); + //Where else is m_types being used? As of now, I dont find any use of it and causing seg fault. So disabling m_types. + //TODO: find where m_types can be used in future and solve the seg fault. + //pt = m_types[t] = new type_info(this,t); + return pt; } -void symbol_table::set_label_address(const symbol *label, unsigned addr) { - std::map<std::string, symbol *>::iterator i = m_symbols.find(label->name()); - assert(i != m_symbols.end()); - symbol *s = i->second; - s->set_label_address(addr); +void symbol_table::set_label_address( const symbol *label, unsigned addr ) +{ + std::map<std::string, symbol *>::iterator i=m_symbols.find(label->name()); + assert( i != m_symbols.end() ); + symbol *s = i->second; + s->set_label_address(addr); } -void symbol_table::dump() { - printf("\n\n"); - printf("Symbol table for \"%s\":\n", m_scope_name.c_str()); - std::map<std::string, symbol *>::iterator i; - for (i = m_symbols.begin(); i != m_symbols.end(); i++) { - printf("%30s : ", i->first.c_str()); - if (i->second) - i->second->print_info(stdout); - else - printf(" <no symbol object> "); - printf("\n"); - } - printf("\n"); +void symbol_table::dump() +{ + printf("\n\n"); + printf("Symbol table for \"%s\":\n", m_scope_name.c_str() ); + std::map<std::string, symbol *>::iterator i; + for( i=m_symbols.begin(); i!=m_symbols.end(); i++ ) { + printf("%30s : ", i->first.c_str() ); + if( i->second ) + i->second->print_info(stdout); + else + printf(" <no symbol object> "); + printf("\n"); + } + printf("\n"); } -unsigned operand_info::get_uid() { - unsigned result = (gpgpu_ctx->operand_info_sm_next_uid)++; - return result; +unsigned operand_info::get_uid() +{ + unsigned result = (gpgpu_ctx->operand_info_sm_next_uid)++; + return result; } -std::list<ptx_instruction *>::iterator -function_info::find_next_real_instruction( - std::list<ptx_instruction *>::iterator i) { - while ((i != m_instructions.end()) && (*i)->is_label()) i++; - return i; +std::list<ptx_instruction*>::iterator function_info::find_next_real_instruction( std::list<ptx_instruction*>::iterator i) +{ + while( (i != m_instructions.end()) && (*i)->is_label() ) + i++; + return i; } -void function_info::create_basic_blocks() { - std::list<ptx_instruction *> leaders; - std::list<ptx_instruction *>::iterator i, l; +void function_info::create_basic_blocks() +{ + std::list<ptx_instruction*> leaders; + std::list<ptx_instruction*>::iterator i, l; - // first instruction is a leader - i = m_instructions.begin(); - leaders.push_back(*i); - i++; - while (i != m_instructions.end()) { - ptx_instruction *pI = *i; - if (pI->is_label()) { - leaders.push_back(pI); - i = find_next_real_instruction(++i); - } else { - switch (pI->get_opcode()) { - case BRA_OP: - case RET_OP: - case EXIT_OP: - case RETP_OP: - case BREAK_OP: - i++; - if (i != m_instructions.end()) leaders.push_back(*i); - i = find_next_real_instruction(i); - break; - case CALL_OP: - case CALLP_OP: - if (pI->has_pred()) { - printf("GPGPU-Sim PTX: Warning found predicated call\n"); + // first instruction is a leader + i=m_instructions.begin(); + leaders.push_back(*i); + i++; + while( i!=m_instructions.end() ) { + ptx_instruction *pI = *i; + if( pI->is_label() ) { + leaders.push_back(pI); + i = find_next_real_instruction(++i); + } else { + switch( pI->get_opcode() ) { + case BRA_OP: case RET_OP: case EXIT_OP: case RETP_OP: case BREAK_OP: i++; - if (i != m_instructions.end()) leaders.push_back(*i); + if( i != m_instructions.end() ) + leaders.push_back(*i); i = find_next_real_instruction(i); - } else + break; + case CALL_OP: case CALLP_OP: + if( pI->has_pred() ) { + printf("GPGPU-Sim PTX: Warning found predicated call\n"); + i++; + if( i != m_instructions.end() ) + leaders.push_back(*i); + i = find_next_real_instruction(i); + } else i++; + break; + default: i++; - break; - default: - i++; - } - } - } + } + } + } - if (leaders.empty()) { - printf("GPGPU-Sim PTX: Function \'%s\' has no basic blocks\n", - m_name.c_str()); - return; - } + if( leaders.empty() ) { + printf("GPGPU-Sim PTX: Function \'%s\' has no basic blocks\n", m_name.c_str()); + return; + } - unsigned bb_id = 0; - l = leaders.begin(); - i = m_instructions.begin(); - m_basic_blocks.push_back( - new basic_block_t(bb_id++, *find_next_real_instruction(i), NULL, 1, 0)); - ptx_instruction *last_real_inst = *(l++); + unsigned bb_id = 0; + l=leaders.begin(); + i=m_instructions.begin(); + m_basic_blocks.push_back( new basic_block_t(bb_id++,*find_next_real_instruction(i),NULL,1,0) ); + ptx_instruction *last_real_inst=*(l++); - for (; i != m_instructions.end(); i++) { - ptx_instruction *pI = *i; - if (l != leaders.end() && *i == *l) { - // found start of next basic block - m_basic_blocks.back()->ptx_end = last_real_inst; - if (find_next_real_instruction(i) != - m_instructions.end()) { // if not bogus trailing label - m_basic_blocks.push_back(new basic_block_t( - bb_id++, *find_next_real_instruction(i), NULL, 0, 0)); - last_real_inst = *find_next_real_instruction(i); + for( ; i!=m_instructions.end(); i++ ) { + ptx_instruction *pI = *i; + if( l != leaders.end() && *i == *l ) { + // found start of next basic block + m_basic_blocks.back()->ptx_end = last_real_inst; + if( find_next_real_instruction(i) != m_instructions.end() ) { // if not bogus trailing label + m_basic_blocks.push_back( new basic_block_t(bb_id++,*find_next_real_instruction(i),NULL,0,0) ); + last_real_inst = *find_next_real_instruction(i); + } + // start search for next leader + l++; } - // start search for next leader - l++; - } - pI->assign_bb(m_basic_blocks.back()); - if (!pI->is_label()) last_real_inst = pI; - } - m_basic_blocks.back()->ptx_end = last_real_inst; - m_basic_blocks.push_back( - /*exit basic block*/ new basic_block_t(bb_id, NULL, NULL, 0, 1)); + pI->assign_bb( m_basic_blocks.back() ); + if( !pI->is_label() ) last_real_inst = pI; + } + m_basic_blocks.back()->ptx_end = last_real_inst; + m_basic_blocks.push_back( /*exit basic block*/ new basic_block_t(bb_id,NULL,NULL,0,1) ); } -void function_info::print_basic_blocks() { - printf("Printing basic blocks for function \'%s\':\n", m_name.c_str()); - std::list<ptx_instruction *>::iterator ptx_itr; - unsigned last_bb = 0; - for (ptx_itr = m_instructions.begin(); ptx_itr != m_instructions.end(); - ptx_itr++) { - if ((*ptx_itr)->get_bb()) { - if ((*ptx_itr)->get_bb()->bb_id != last_bb) { - printf("\n"); - last_bb = (*ptx_itr)->get_bb()->bb_id; +void function_info::print_basic_blocks() +{ + printf("Printing basic blocks for function \'%s\':\n", m_name.c_str() ); + std::list<ptx_instruction*>::iterator ptx_itr; + unsigned last_bb=0; + for (ptx_itr = m_instructions.begin();ptx_itr != m_instructions.end(); ptx_itr++) { + if( (*ptx_itr)->get_bb() ) { + if( (*ptx_itr)->get_bb()->bb_id != last_bb ) { + printf("\n"); + last_bb = (*ptx_itr)->get_bb()->bb_id; + } + printf("bb_%02u\t: ", (*ptx_itr)->get_bb()->bb_id); + (*ptx_itr)->print_insn(); + printf("\n"); } - printf("bb_%02u\t: ", (*ptx_itr)->get_bb()->bb_id); - (*ptx_itr)->print_insn(); + } + printf("\nSummary of basic blocks for \'%s\':\n", m_name.c_str() ); + std::vector<basic_block_t*>::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { + printf("bb_%02u\t:", (*bb_itr)->bb_id); + if ((*bb_itr)->ptx_begin) + printf(" first: %s\t", ((*bb_itr)->ptx_begin)->get_opcode_cstr()); + else printf(" first: NULL\t"); + if ((*bb_itr)->ptx_end) { + printf(" last: %s\t", ((*bb_itr)->ptx_end)->get_opcode_cstr()); + } else printf(" last: NULL\t"); printf("\n"); - } - } - printf("\nSummary of basic blocks for \'%s\':\n", m_name.c_str()); - std::vector<basic_block_t *>::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); - bb_itr++) { - printf("bb_%02u\t:", (*bb_itr)->bb_id); - if ((*bb_itr)->ptx_begin) - printf(" first: %s\t", ((*bb_itr)->ptx_begin)->get_opcode_cstr()); - else - printf(" first: NULL\t"); - if ((*bb_itr)->ptx_end) { - printf(" last: %s\t", ((*bb_itr)->ptx_end)->get_opcode_cstr()); - } else - printf(" last: NULL\t"); - printf("\n"); - } - printf("\n"); + } + printf("\n"); } -void function_info::print_basic_block_links() { - printf("Printing basic blocks links for function \'%s\':\n", m_name.c_str()); - std::vector<basic_block_t *>::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); - bb_itr++) { - printf("ID: %d\t:", (*bb_itr)->bb_id); - if (!(*bb_itr)->predecessor_ids.empty()) { - printf("Predecessors:"); - std::set<int>::iterator p; - for (p = (*bb_itr)->predecessor_ids.begin(); - p != (*bb_itr)->predecessor_ids.end(); p++) { - printf(" %d", *p); +void function_info::print_basic_block_links() +{ + printf("Printing basic blocks links for function \'%s\':\n", m_name.c_str() ); + std::vector<basic_block_t*>::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { + printf("ID: %d\t:", (*bb_itr)->bb_id); + if ( !(*bb_itr)->predecessor_ids.empty() ) { + printf("Predecessors:"); + std::set<int>::iterator p; + for (p= (*bb_itr)->predecessor_ids.begin();p != (*bb_itr)->predecessor_ids.end();p++) { + printf(" %d", *p); + } + printf("\t"); } - printf("\t"); - } - if (!(*bb_itr)->successor_ids.empty()) { - printf("Successors:"); - std::set<int>::iterator s; - for (s = (*bb_itr)->successor_ids.begin(); - s != (*bb_itr)->successor_ids.end(); s++) { - printf(" %d", *s); + if ( !(*bb_itr)->successor_ids.empty() ) { + printf("Successors:"); + std::set<int>::iterator s; + for (s= (*bb_itr)->successor_ids.begin();s != (*bb_itr)->successor_ids.end();s++) { + printf(" %d", *s); + } } - } - printf("\n"); - } + printf("\n"); + } } -operand_info *function_info::find_break_target( - ptx_instruction *p_break_insn) // find the target of a break instruction +operand_info* function_info::find_break_target( ptx_instruction * p_break_insn ) //find the target of a break instruction { - const basic_block_t *break_bb = p_break_insn->get_bb(); - // go through the dominator tree - for (const basic_block_t *p_bb = break_bb; p_bb->immediatedominator_id != -1; - p_bb = m_basic_blocks[p_bb->immediatedominator_id]) { - // reverse search through instructions in basic block for breakaddr - // instruction - unsigned insn_addr = p_bb->ptx_end->get_m_instr_mem_index(); - while (insn_addr >= p_bb->ptx_begin->get_m_instr_mem_index()) { - ptx_instruction *pI = m_instr_mem[insn_addr]; - insn_addr -= 1; - if (pI == NULL) - continue; // temporary solution for variable size instructions - if (pI->get_opcode() == BREAKADDR_OP) { - return &(pI->dst()); + const basic_block_t *break_bb = p_break_insn->get_bb(); + // go through the dominator tree + for(const basic_block_t *p_bb = break_bb; + p_bb->immediatedominator_id != -1; + p_bb = m_basic_blocks[p_bb->immediatedominator_id]) + { + // reverse search through instructions in basic block for breakaddr instruction + unsigned insn_addr = p_bb->ptx_end->get_m_instr_mem_index(); + while (insn_addr >= p_bb->ptx_begin->get_m_instr_mem_index()) { + ptx_instruction *pI = m_instr_mem[insn_addr]; + insn_addr -= 1; + if (pI == NULL) continue; // temporary solution for variable size instructions + if (pI->get_opcode() == BREAKADDR_OP) { + return &(pI->dst()); + } } - } - } + } - assert(0); + assert(0); - // lazy fallback: just traverse backwards? - for (int insn_addr = p_break_insn->get_m_instr_mem_index(); insn_addr >= 0; - insn_addr--) { - ptx_instruction *pI = m_instr_mem[insn_addr]; - if (pI->get_opcode() == BREAKADDR_OP) { - return &(pI->dst()); - } - } + // lazy fallback: just traverse backwards? + for (int insn_addr = p_break_insn->get_m_instr_mem_index(); + insn_addr >= 0; insn_addr--) + { + ptx_instruction *pI = m_instr_mem[insn_addr]; + if (pI->get_opcode() == BREAKADDR_OP) { + return &(pI->dst()); + } + } - return NULL; + return NULL; } -void function_info::connect_basic_blocks() // iterate across m_basic_blocks of - // function, connecting basic blocks - // together +void function_info::connect_basic_blocks( ) //iterate across m_basic_blocks of function, connecting basic blocks together { - std::vector<basic_block_t *>::iterator bb_itr; - std::vector<basic_block_t *>::iterator bb_target_itr; - basic_block_t *exit_bb = m_basic_blocks.back(); + std::vector<basic_block_t*>::iterator bb_itr; + std::vector<basic_block_t*>::iterator bb_target_itr; + basic_block_t* exit_bb = m_basic_blocks.back(); - // start from first basic block, which we know is the entry point - bb_itr = m_basic_blocks.begin(); - for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); - bb_itr++) { - ptx_instruction *pI = (*bb_itr)->ptx_end; - if ((*bb_itr)->is_exit) // reached last basic block, no successors to link - continue; - if (pI->get_opcode() == RETP_OP || pI->get_opcode() == RET_OP || - pI->get_opcode() == EXIT_OP) { - (*bb_itr)->successor_ids.insert(exit_bb->bb_id); - exit_bb->predecessor_ids.insert((*bb_itr)->bb_id); - if (pI->has_pred()) { - printf("GPGPU-Sim PTX: Warning detected predicated return/exit.\n"); - // if predicated, add link to next block - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - if (next_addr < m_instr_mem_size && m_instr_mem[next_addr]) { - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - (*bb_itr)->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } - } - continue; - } else if (pI->get_opcode() == BRA_OP) { - // find successor and link that basic_block to this one - operand_info &target = pI->dst(); // get operand, e.g. target name - unsigned addr = labels[target.name()]; - ptx_instruction *target_pI = m_instr_mem[addr]; - basic_block_t *target_bb = target_pI->get_bb(); - (*bb_itr)->successor_ids.insert(target_bb->bb_id); - target_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } + //start from first basic block, which we know is the entry point + bb_itr = m_basic_blocks.begin(); + for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { + ptx_instruction *pI = (*bb_itr)->ptx_end; + if ((*bb_itr)->is_exit) //reached last basic block, no successors to link + continue; + if (pI->get_opcode() == RETP_OP || pI->get_opcode() == RET_OP || pI->get_opcode() == EXIT_OP ) { + (*bb_itr)->successor_ids.insert(exit_bb->bb_id); + exit_bb->predecessor_ids.insert((*bb_itr)->bb_id); + if( pI->has_pred() ) { + printf("GPGPU-Sim PTX: Warning detected predicated return/exit.\n"); + // if predicated, add link to next block + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + if( next_addr < m_instr_mem_size && m_instr_mem[next_addr] ) { + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + (*bb_itr)->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } + } + continue; + } else if (pI->get_opcode() == BRA_OP) { + //find successor and link that basic_block to this one + operand_info &target = pI->dst(); //get operand, e.g. target name + unsigned addr = labels[ target.name() ]; + ptx_instruction *target_pI = m_instr_mem[addr]; + basic_block_t *target_bb = target_pI->get_bb(); + (*bb_itr)->successor_ids.insert(target_bb->bb_id); + target_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } - if (!(pI->get_opcode() == BRA_OP && (!pI->has_pred()))) { - // if basic block does not end in an unpredicated branch, - // then next basic block is also successor - // (this is better than testing for .uni) - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - (*bb_itr)->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } else - assert(pI->get_opcode() == BRA_OP); - } + if ( !(pI->get_opcode()==BRA_OP && (!pI->has_pred())) ) { + // if basic block does not end in an unpredicated branch, + // then next basic block is also successor + // (this is better than testing for .uni) + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + (*bb_itr)->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } else + assert(pI->get_opcode() == BRA_OP); + } } -bool function_info::connect_break_targets() // connecting break instructions - // with proper targets +bool function_info::connect_break_targets() //connecting break instructions with proper targets { - std::vector<basic_block_t *>::iterator bb_itr; - std::vector<basic_block_t *>::iterator bb_target_itr; - bool modified = false; + std::vector<basic_block_t*>::iterator bb_itr; + std::vector<basic_block_t*>::iterator bb_target_itr; + bool modified = false; - // start from first basic block, which we know is the entry point - bb_itr = m_basic_blocks.begin(); - for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); - bb_itr++) { - basic_block_t *p_bb = *bb_itr; - ptx_instruction *pI = p_bb->ptx_end; - if (p_bb->is_exit) // reached last basic block, no successors to link - continue; - if (pI->get_opcode() == BREAK_OP) { - // backup existing successor_ids for stability check - std::set<int> orig_successor_ids = p_bb->successor_ids; + //start from first basic block, which we know is the entry point + bb_itr = m_basic_blocks.begin(); + for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { + basic_block_t *p_bb = *bb_itr; + ptx_instruction *pI = p_bb->ptx_end; + if (p_bb->is_exit) //reached last basic block, no successors to link + continue; + if (pI->get_opcode() == BREAK_OP) { + // backup existing successor_ids for stability check + std::set<int> orig_successor_ids = p_bb->successor_ids; - // erase the previous linkage with old successors - for (std::set<int>::iterator succ_ids = p_bb->successor_ids.begin(); - succ_ids != p_bb->successor_ids.end(); ++succ_ids) { - basic_block_t *successor_bb = m_basic_blocks[*succ_ids]; - successor_bb->predecessor_ids.erase(p_bb->bb_id); - } - p_bb->successor_ids.clear(); + // erase the previous linkage with old successors + for(std::set<int>::iterator succ_ids = p_bb->successor_ids.begin(); succ_ids != p_bb->successor_ids.end(); ++succ_ids) { + basic_block_t *successor_bb = m_basic_blocks[*succ_ids]; + successor_bb->predecessor_ids.erase(p_bb->bb_id); + } + p_bb->successor_ids.clear(); - // find successor and link that basic_block to this one - // successor of a break is set by an preceeding breakaddr instruction - operand_info *target = find_break_target(pI); - unsigned addr = labels[target->name()]; - ptx_instruction *target_pI = m_instr_mem[addr]; - basic_block_t *target_bb = target_pI->get_bb(); - p_bb->successor_ids.insert(target_bb->bb_id); - target_bb->predecessor_ids.insert(p_bb->bb_id); + //find successor and link that basic_block to this one + //successor of a break is set by an preceeding breakaddr instruction + operand_info *target = find_break_target(pI); + unsigned addr = labels[ target->name() ]; + ptx_instruction *target_pI = m_instr_mem[addr]; + basic_block_t *target_bb = target_pI->get_bb(); + p_bb->successor_ids.insert(target_bb->bb_id); + target_bb->predecessor_ids.insert(p_bb->bb_id); - if (pI->has_pred()) { - // predicated break - add link to next basic block - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - p_bb->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert(p_bb->bb_id); - } + if (pI->has_pred()) { + // predicated break - add link to next basic block + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + p_bb->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert(p_bb->bb_id); + } - modified = modified || (orig_successor_ids != p_bb->successor_ids); - } - } + modified = modified || (orig_successor_ids != p_bb->successor_ids); + } + } - return modified; + return modified; } -void function_info::do_pdom() { - create_basic_blocks(); - connect_basic_blocks(); - bool modified = false; - do { - find_dominators(); - find_idominators(); - modified = connect_break_targets(); - } while (modified == true); +void function_info::do_pdom() +{ + create_basic_blocks(); + connect_basic_blocks(); + bool modified = false; + do { + find_dominators(); + find_idominators(); + modified = connect_break_targets(); + } while (modified == true); - if (g_debug_execution >= 50) { - print_basic_blocks(); - print_basic_block_links(); - print_basic_block_dot(); - } - if (g_debug_execution >= 2) { - print_dominators(); - } - find_postdominators(); - find_ipostdominators(); - if (g_debug_execution >= 50) { - print_postdominators(); - print_ipostdominators(); - } - printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", - m_name.c_str()); - for (unsigned ii = 0; ii < m_n; - ii += m_instr_mem[ii]->inst_size()) { // handle branch instructions - ptx_instruction *pI = m_instr_mem[ii]; - pI->pre_decode(); - } - printf("GPGPU-Sim PTX: ... done pre-decoding instructions for \'%s\'.\n", - m_name.c_str()); - fflush(stdout); - m_assembled = true; + if ( g_debug_execution>=50 ) { + print_basic_blocks(); + print_basic_block_links(); + print_basic_block_dot(); + } + if ( g_debug_execution>=2 ) { + print_dominators(); + } + find_postdominators(); + find_ipostdominators(); + if ( g_debug_execution>=50 ) { + print_postdominators(); + print_ipostdominators(); + } + printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", m_name.c_str() ); + for ( unsigned ii=0; ii < m_n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions + ptx_instruction *pI = m_instr_mem[ii]; + pI->pre_decode(); + } + printf("GPGPU-Sim PTX: ... done pre-decoding instructions for \'%s\'.\n", m_name.c_str() ); + fflush(stdout); + m_assembled = true; } -void intersect(std::set<int> &A, const std::set<int> &B) { - // return intersection of A and B in A - for (std::set<int>::iterator a = A.begin(); a != A.end();) { - std::set<int>::iterator a_next = a; - a_next++; - if (B.find(*a) == B.end()) { - A.erase(*a); - a = a_next; - } else - a++; - } +void intersect( std::set<int> &A, const std::set<int> &B ) +{ + // return intersection of A and B in A + for( std::set<int>::iterator a=A.begin(); a!=A.end(); ) { + std::set<int>::iterator a_next = a; + a_next++; + if( B.find(*a) == B.end() ) { + A.erase(*a); + a = a_next; + } else + a++; + } } -bool is_equal(const std::set<int> &A, const std::set<int> &B) { - if (A.size() != B.size()) return false; - for (std::set<int>::iterator b = B.begin(); b != B.end(); b++) - if (A.find(*b) == A.end()) return false; - return true; +bool is_equal( const std::set<int> &A, const std::set<int> &B ) +{ + if( A.size() != B.size() ) + return false; + for( std::set<int>::iterator b=B.begin(); b!=B.end(); b++ ) + if( A.find(*b) == A.end() ) + return false; + return true; } -void print_set(const std::set<int> &A) { - std::set<int>::iterator a; - for (a = A.begin(); a != A.end(); a++) { - printf("%d ", (*a)); - } - printf("\n"); +void print_set(const std::set<int> &A) +{ + std::set<int>::iterator a; + for (a= A.begin(); a != A.end(); a++) { + printf("%d ", (*a)); + } + printf("\n"); } -void function_info::find_dominators() { - // find dominators using algorithm of Muchnick's Adv. Compiler Design & - // Implemmntation Fig 7.14 - printf("GPGPU-Sim PTX: Finding dominators for \'%s\'...\n", m_name.c_str()); - fflush(stdout); - assert(m_basic_blocks.size() >= 2); // must have a distinquished entry block - std::vector<basic_block_t *>::iterator bb_itr = m_basic_blocks.begin(); - (*bb_itr)->dominator_ids.insert( - (*bb_itr)->bb_id); // the only dominator of the entry block is the entry - // copy all basic blocks to all dominator lists EXCEPT for the entry block - for (++bb_itr; bb_itr != m_basic_blocks.end(); bb_itr++) { - for (unsigned i = 0; i < m_basic_blocks.size(); i++) - (*bb_itr)->dominator_ids.insert(i); - } - bool change = true; - while (change) { - change = false; - for (int h = 1 /*skip entry*/; h < m_basic_blocks.size(); ++h) { - assert(m_basic_blocks[h]->bb_id == (unsigned)h); - std::set<int> T; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) T.insert(i); - for (std::set<int>::iterator s = - m_basic_blocks[h]->predecessor_ids.begin(); - s != m_basic_blocks[h]->predecessor_ids.end(); s++) - intersect(T, m_basic_blocks[*s]->dominator_ids); - T.insert(h); - if (!is_equal(T, m_basic_blocks[h]->dominator_ids)) { - change = true; - m_basic_blocks[h]->dominator_ids = T; +void function_info::find_dominators( ) +{ + // find dominators using algorithm of Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 + printf("GPGPU-Sim PTX: Finding dominators for \'%s\'...\n", m_name.c_str() ); + fflush(stdout); + assert( m_basic_blocks.size() >= 2 ); // must have a distinquished entry block + std::vector<basic_block_t*>::iterator bb_itr = m_basic_blocks.begin(); + (*bb_itr)->dominator_ids.insert((*bb_itr)->bb_id); // the only dominator of the entry block is the entry + //copy all basic blocks to all dominator lists EXCEPT for the entry block + for (++bb_itr;bb_itr != m_basic_blocks.end(); bb_itr++) { + for (unsigned i = 0; i < m_basic_blocks.size(); i++) + (*bb_itr)->dominator_ids.insert(i); + } + bool change = true; + while (change) { + change = false; + for ( int h = 1/*skip entry*/; h < m_basic_blocks.size(); ++h ) { + assert( m_basic_blocks[h]->bb_id == (unsigned)h ); + std::set<int> T; + for (unsigned i=0;i< m_basic_blocks.size();i++) + T.insert(i); + for ( std::set<int>::iterator s = m_basic_blocks[h]->predecessor_ids.begin();s != m_basic_blocks[h]->predecessor_ids.end();s++) + intersect(T, m_basic_blocks[*s]->dominator_ids); + T.insert(h); + if (!is_equal(T, m_basic_blocks[h]->dominator_ids)) { + change = true; + m_basic_blocks[h]->dominator_ids = T; + } } - } - } - // clean the basic block of dominators of it has no predecessors -- except for - // entry block - bb_itr = m_basic_blocks.begin(); - for (++bb_itr; bb_itr != m_basic_blocks.end(); bb_itr++) { - if ((*bb_itr)->predecessor_ids.empty()) (*bb_itr)->dominator_ids.clear(); - } + } + //clean the basic block of dominators of it has no predecessors -- except for entry block + bb_itr = m_basic_blocks.begin(); + for (++bb_itr;bb_itr != m_basic_blocks.end(); bb_itr++) { + if ((*bb_itr)->predecessor_ids.empty()) + (*bb_itr)->dominator_ids.clear(); + } } -void function_info::find_postdominators() { - // find postdominators using algorithm of Muchnick's Adv. Compiler Design & - // Implemmntation Fig 7.14 - printf("GPGPU-Sim PTX: Finding postdominators for \'%s\'...\n", - m_name.c_str()); - fflush(stdout); - assert(m_basic_blocks.size() >= 2); // must have a distinquished exit block - std::vector<basic_block_t *>::reverse_iterator bb_itr = - m_basic_blocks.rbegin(); - (*bb_itr)->postdominator_ids.insert( - (*bb_itr) - ->bb_id); // the only postdominator of the exit block is the exit - for (++bb_itr; bb_itr != m_basic_blocks.rend(); - bb_itr++) { // copy all basic blocks to all postdominator lists EXCEPT - // for the exit block - for (unsigned i = 0; i < m_basic_blocks.size(); i++) - (*bb_itr)->postdominator_ids.insert(i); - } - bool change = true; - while (change) { - change = false; - for (int h = m_basic_blocks.size() - 2 /*skip exit*/; h >= 0; --h) { - assert(m_basic_blocks[h]->bb_id == (unsigned)h); - std::set<int> T; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) T.insert(i); - for (std::set<int>::iterator s = m_basic_blocks[h]->successor_ids.begin(); - s != m_basic_blocks[h]->successor_ids.end(); s++) - intersect(T, m_basic_blocks[*s]->postdominator_ids); - T.insert(h); - if (!is_equal(T, m_basic_blocks[h]->postdominator_ids)) { - change = true; - m_basic_blocks[h]->postdominator_ids = T; +void function_info::find_postdominators( ) +{ + // find postdominators using algorithm of Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 + printf("GPGPU-Sim PTX: Finding postdominators for \'%s\'...\n", m_name.c_str() ); + fflush(stdout); + assert( m_basic_blocks.size() >= 2 ); // must have a distinquished exit block + std::vector<basic_block_t*>::reverse_iterator bb_itr = m_basic_blocks.rbegin(); + (*bb_itr)->postdominator_ids.insert((*bb_itr)->bb_id); // the only postdominator of the exit block is the exit + for (++bb_itr;bb_itr != m_basic_blocks.rend();bb_itr++) { //copy all basic blocks to all postdominator lists EXCEPT for the exit block + for (unsigned i=0; i<m_basic_blocks.size(); i++) + (*bb_itr)->postdominator_ids.insert(i); + } + bool change = true; + while (change) { + change = false; + for ( int h = m_basic_blocks.size()-2/*skip exit*/; h >= 0 ; --h ) { + assert( m_basic_blocks[h]->bb_id == (unsigned)h ); + std::set<int> T; + for (unsigned i=0;i< m_basic_blocks.size();i++) + T.insert(i); + for ( std::set<int>::iterator s = m_basic_blocks[h]->successor_ids.begin();s != m_basic_blocks[h]->successor_ids.end();s++) + intersect(T, m_basic_blocks[*s]->postdominator_ids); + T.insert(h); + if (!is_equal(T,m_basic_blocks[h]->postdominator_ids)) { + change = true; + m_basic_blocks[h]->postdominator_ids = T; + } } - } - } + } } -void function_info::find_ipostdominators() { - // find immediate postdominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - printf("GPGPU-Sim PTX: Finding immediate postdominators for \'%s\'...\n", - m_name.c_str()); - fflush(stdout); - assert(m_basic_blocks.size() >= 2); // must have a distinquished exit block - for (unsigned i = 0; i < m_basic_blocks.size(); - i++) { // initialize Tmp(n) to all pdoms of n except for n - m_basic_blocks[i]->Tmp_ids = m_basic_blocks[i]->postdominator_ids; - assert(m_basic_blocks[i]->bb_id == i); - m_basic_blocks[i]->Tmp_ids.erase(i); - } - for (int n = m_basic_blocks.size() - 2; n >= 0; --n) { - // point iterator to basic block before the exit - for (std::set<int>::iterator s = m_basic_blocks[n]->Tmp_ids.begin(); - s != m_basic_blocks[n]->Tmp_ids.end(); s++) { - int bb_s = *s; - for (std::set<int>::iterator t = m_basic_blocks[n]->Tmp_ids.begin(); - t != m_basic_blocks[n]->Tmp_ids.end();) { - std::set<int>::iterator t_next = t; - t_next++; // might erase thing pointed to be t, invalidating iterator t - if (*s == *t) { - t = t_next; - continue; - } - int bb_t = *t; - if (m_basic_blocks[bb_s]->postdominator_ids.find(bb_t) != - m_basic_blocks[bb_s]->postdominator_ids.end()) - m_basic_blocks[n]->Tmp_ids.erase(bb_t); - t = t_next; +void function_info::find_ipostdominators( ) +{ + // find immediate postdominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + printf("GPGPU-Sim PTX: Finding immediate postdominators for \'%s\'...\n", m_name.c_str() ); + fflush(stdout); + assert( m_basic_blocks.size() >= 2 ); // must have a distinquished exit block + for (unsigned i=0; i<m_basic_blocks.size(); i++) { //initialize Tmp(n) to all pdoms of n except for n + m_basic_blocks[i]->Tmp_ids = m_basic_blocks[i]->postdominator_ids; + assert( m_basic_blocks[i]->bb_id == i ); + m_basic_blocks[i]->Tmp_ids.erase(i); + } + for ( int n = m_basic_blocks.size()-2; n >=0;--n) { + // point iterator to basic block before the exit + for( std::set<int>::iterator s=m_basic_blocks[n]->Tmp_ids.begin(); s != m_basic_blocks[n]->Tmp_ids.end(); s++ ) { + int bb_s = *s; + for( std::set<int>::iterator t=m_basic_blocks[n]->Tmp_ids.begin(); t != m_basic_blocks[n]->Tmp_ids.end(); ) { + std::set<int>::iterator t_next = t; t_next++; // might erase thing pointed to be t, invalidating iterator t + if( *s == *t ) { + t = t_next; + continue; + } + int bb_t = *t; + if( m_basic_blocks[bb_s]->postdominator_ids.find(bb_t) != m_basic_blocks[bb_s]->postdominator_ids.end() ) + m_basic_blocks[n]->Tmp_ids.erase(bb_t); + t = t_next; + } } - } - } - unsigned num_ipdoms = 0; - for (int n = m_basic_blocks.size() - 1; n >= 0; --n) { - assert(m_basic_blocks[n]->Tmp_ids.size() <= 1); - // if the above assert fails we have an error in either postdominator - // computation, the flow graph does not have a unique exit, or some other - // error - if (!m_basic_blocks[n]->Tmp_ids.empty()) { - m_basic_blocks[n]->immediatepostdominator_id = - *m_basic_blocks[n]->Tmp_ids.begin(); - num_ipdoms++; - } - } - assert(num_ipdoms == m_basic_blocks.size() - 1); - // the exit node does not have an immediate post dominator, but everyone else - // should + } + unsigned num_ipdoms=0; + for ( int n = m_basic_blocks.size()-1; n >=0;--n) { + assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); + // if the above assert fails we have an error in either postdominator + // computation, the flow graph does not have a unique exit, or some other error + if( !m_basic_blocks[n]->Tmp_ids.empty() ) { + m_basic_blocks[n]->immediatepostdominator_id = *m_basic_blocks[n]->Tmp_ids.begin(); + num_ipdoms++; + } + } + assert( num_ipdoms == m_basic_blocks.size()-1 ); + // the exit node does not have an immediate post dominator, but everyone else should } -void function_info::find_idominators() { - // find immediate dominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - printf("GPGPU-Sim PTX: Finding immediate dominators for \'%s\'...\n", - m_name.c_str()); - fflush(stdout); - assert(m_basic_blocks.size() >= 2); // must have a distinquished entry block - for (unsigned i = 0; i < m_basic_blocks.size(); - i++) { // initialize Tmp(n) to all doms of n except for n - m_basic_blocks[i]->Tmp_ids = m_basic_blocks[i]->dominator_ids; - assert(m_basic_blocks[i]->bb_id == i); - m_basic_blocks[i]->Tmp_ids.erase(i); - } - for (int n = 0; n < m_basic_blocks.size(); ++n) { - // point iterator to basic block before the exit - for (std::set<int>::iterator s = m_basic_blocks[n]->Tmp_ids.begin(); - s != m_basic_blocks[n]->Tmp_ids.end(); s++) { - int bb_s = *s; - for (std::set<int>::iterator t = m_basic_blocks[n]->Tmp_ids.begin(); - t != m_basic_blocks[n]->Tmp_ids.end();) { - std::set<int>::iterator t_next = t; - t_next++; // might erase thing pointed to be t, invalidating iterator t - if (*s == *t) { - t = t_next; - continue; - } - int bb_t = *t; - if (m_basic_blocks[bb_s]->dominator_ids.find(bb_t) != - m_basic_blocks[bb_s]->dominator_ids.end()) - m_basic_blocks[n]->Tmp_ids.erase(bb_t); - t = t_next; +void function_info::find_idominators( ) +{ + // find immediate dominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + printf("GPGPU-Sim PTX: Finding immediate dominators for \'%s\'...\n", m_name.c_str() ); + fflush(stdout); + assert( m_basic_blocks.size() >= 2 ); // must have a distinquished entry block + for (unsigned i=0; i<m_basic_blocks.size(); i++) { //initialize Tmp(n) to all doms of n except for n + m_basic_blocks[i]->Tmp_ids = m_basic_blocks[i]->dominator_ids; + assert( m_basic_blocks[i]->bb_id == i ); + m_basic_blocks[i]->Tmp_ids.erase(i); + } + for ( int n = 0; n < m_basic_blocks.size(); ++n) { + // point iterator to basic block before the exit + for( std::set<int>::iterator s=m_basic_blocks[n]->Tmp_ids.begin(); s != m_basic_blocks[n]->Tmp_ids.end(); s++ ) { + int bb_s = *s; + for( std::set<int>::iterator t=m_basic_blocks[n]->Tmp_ids.begin(); t != m_basic_blocks[n]->Tmp_ids.end(); ) { + std::set<int>::iterator t_next = t; t_next++; // might erase thing pointed to be t, invalidating iterator t + if( *s == *t ) { + t = t_next; + continue; + } + int bb_t = *t; + if( m_basic_blocks[bb_s]->dominator_ids.find(bb_t) != m_basic_blocks[bb_s]->dominator_ids.end() ) + m_basic_blocks[n]->Tmp_ids.erase(bb_t); + t = t_next; + } } - } - } - unsigned num_idoms = 0; - unsigned num_nopred = 0; - for (int n = 0; n < m_basic_blocks.size(); ++n) { - // assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); - // if the above assert fails we have an error in either dominator - // computation, the flow graph does not have a unique entry, or some other - // error - if (!m_basic_blocks[n]->Tmp_ids.empty()) { - m_basic_blocks[n]->immediatedominator_id = - *m_basic_blocks[n]->Tmp_ids.begin(); - num_idoms++; - } else if (m_basic_blocks[n]->predecessor_ids.empty()) { - num_nopred += 1; - } - } - assert(num_idoms == m_basic_blocks.size() - num_nopred); - // the entry node does not have an immediate dominator, but everyone else - // should + } + unsigned num_idoms=0; + unsigned num_nopred = 0; + for ( int n = 0; n < m_basic_blocks.size(); ++n) { + //assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); + // if the above assert fails we have an error in either dominator + // computation, the flow graph does not have a unique entry, or some other error + if( !m_basic_blocks[n]->Tmp_ids.empty() ) { + m_basic_blocks[n]->immediatedominator_id = *m_basic_blocks[n]->Tmp_ids.begin(); + num_idoms++; + } else if (m_basic_blocks[n]->predecessor_ids.empty()) { + num_nopred += 1; + } + } + assert( num_idoms == m_basic_blocks.size()-num_nopred ); + // the entry node does not have an immediate dominator, but everyone else should } -void function_info::print_dominators() { - printf("Printing dominators for function \'%s\':\n", m_name.c_str()); - std::vector<int>::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - for (std::set<int>::iterator j = m_basic_blocks[i]->dominator_ids.begin(); - j != m_basic_blocks[i]->dominator_ids.end(); j++) - printf(" %d", *j); - printf("\n"); - } +void function_info::print_dominators() +{ + printf("Printing dominators for function \'%s\':\n", m_name.c_str() ); + std::vector<int>::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + for( std::set<int>::iterator j=m_basic_blocks[i]->dominator_ids.begin(); j!=m_basic_blocks[i]->dominator_ids.end(); j++) + printf(" %d", *j ); + printf("\n"); + } } -void function_info::print_postdominators() { - printf("Printing postdominators for function \'%s\':\n", m_name.c_str()); - std::vector<int>::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - for (std::set<int>::iterator j = - m_basic_blocks[i]->postdominator_ids.begin(); - j != m_basic_blocks[i]->postdominator_ids.end(); j++) - printf(" %d", *j); - printf("\n"); - } +void function_info::print_postdominators() +{ + printf("Printing postdominators for function \'%s\':\n", m_name.c_str() ); + std::vector<int>::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + for( std::set<int>::iterator j=m_basic_blocks[i]->postdominator_ids.begin(); j!=m_basic_blocks[i]->postdominator_ids.end(); j++) + printf(" %d", *j ); + printf("\n"); + } } -void function_info::print_ipostdominators() { - printf("Printing immediate postdominators for function \'%s\':\n", - m_name.c_str()); - std::vector<int>::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - printf("%d\n", m_basic_blocks[i]->immediatepostdominator_id); - } +void function_info::print_ipostdominators() +{ + printf("Printing immediate postdominators for function \'%s\':\n", m_name.c_str() ); + std::vector<int>::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + printf("%d\n", m_basic_blocks[i]->immediatepostdominator_id); + } } -void function_info::print_idominators() { - printf("Printing immediate dominators for function \'%s\':\n", - m_name.c_str()); - std::vector<int>::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - printf("%d\n", m_basic_blocks[i]->immediatedominator_id); - } +void function_info::print_idominators() +{ + printf("Printing immediate dominators for function \'%s\':\n", m_name.c_str() ); + std::vector<int>::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + printf("%d\n", m_basic_blocks[i]->immediatedominator_id); + } } -unsigned function_info::get_num_reconvergence_pairs() { - if (!num_reconvergence_pairs) { - if (m_basic_blocks.size() == 0) return 0; - for (unsigned i = 0; i < (m_basic_blocks.size() - 1); - i++) { // last basic block containing exit obviously won't have a pair - if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { - num_reconvergence_pairs++; +unsigned function_info::get_num_reconvergence_pairs() +{ + if (!num_reconvergence_pairs) { + if( m_basic_blocks.size() == 0 ) + return 0; + for (unsigned i=0; i< (m_basic_blocks.size()-1); i++) { //last basic block containing exit obviously won't have a pair + if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { + num_reconvergence_pairs++; + } } - } - } - return num_reconvergence_pairs; + } + return num_reconvergence_pairs; } -void function_info::get_reconvergence_pairs(gpgpu_recon_t *recon_points) { - unsigned idx = 0; // array index - if (m_basic_blocks.size() == 0) return; - for (unsigned i = 0; i < (m_basic_blocks.size() - 1); - i++) { // last basic block containing exit obviously won't have a pair +void function_info::get_reconvergence_pairs(gpgpu_recon_t* recon_points) +{ + unsigned idx=0; //array index + if( m_basic_blocks.size() == 0 ) + return; + for (unsigned i=0; i< (m_basic_blocks.size()-1); i++) { //last basic block containing exit obviously won't have a pair #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("i=%d\n", i); - fflush(stdout); + printf("i=%d\n", i); fflush(stdout); #endif - if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { + if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("\tbranch!\n"); - printf("\tbb_id=%d; ipdom=%d\n", m_basic_blocks[i]->bb_id, - m_basic_blocks[i]->immediatepostdominator_id); - printf("\tm_instr_mem index=%d\n", - m_basic_blocks[i]->ptx_end->get_m_instr_mem_index()); - fflush(stdout); + printf("\tbranch!\n"); + printf("\tbb_id=%d; ipdom=%d\n", m_basic_blocks[i]->bb_id, m_basic_blocks[i]->immediatepostdominator_id); + printf("\tm_instr_mem index=%d\n", m_basic_blocks[i]->ptx_end->get_m_instr_mem_index()); + fflush(stdout); #endif - recon_points[idx].source_pc = m_basic_blocks[i]->ptx_end->get_PC(); - recon_points[idx].source_inst = m_basic_blocks[i]->ptx_end; + recon_points[idx].source_pc = m_basic_blocks[i]->ptx_end->get_PC(); + recon_points[idx].source_inst = m_basic_blocks[i]->ptx_end; #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("\trecon_points[idx].source_pc=%d\n", recon_points[idx].source_pc); + printf("\trecon_points[idx].source_pc=%d\n", recon_points[idx].source_pc); #endif - if (m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] - ->ptx_begin) { - recon_points[idx].target_pc = - m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] - ->ptx_begin->get_PC(); - recon_points[idx].target_inst = - m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] - ->ptx_begin; - } else { - // reconverge after function return - recon_points[idx].target_pc = -2; - recon_points[idx].target_inst = NULL; - } + if( m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin ) { + recon_points[idx].target_pc = m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin->get_PC(); + recon_points[idx].target_inst = m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin; + } else { + // reconverge after function return + recon_points[idx].target_pc = -2; + recon_points[idx].target_inst = NULL; + } #ifdef DEBUG_GET_RECONVERG_PAIRS - m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] - ->ptx_begin->print_insn(); - printf("\trecon_points[idx].target_pc=%d\n", recon_points[idx].target_pc); - fflush(stdout); + m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin->print_insn(); + printf("\trecon_points[idx].target_pc=%d\n", recon_points[idx].target_pc); fflush(stdout); #endif - idx++; - } - } + idx++; + } + } } // interface with graphviz (print the graph in DOT language) for plotting -void function_info::print_basic_block_dot() { - printf("Basic Block in DOT\n"); - printf("digraph %s {\n", m_name.c_str()); - std::vector<basic_block_t *>::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); - bb_itr++) { - printf("\t"); - std::set<int>::iterator s; - for (s = (*bb_itr)->successor_ids.begin(); - s != (*bb_itr)->successor_ids.end(); s++) { - unsigned succ_bb = *s; - printf("%d -> %d; ", (*bb_itr)->bb_id, succ_bb); - } - printf("\n"); - } - printf("}\n"); +void function_info::print_basic_block_dot() +{ + printf("Basic Block in DOT\n"); + printf("digraph %s {\n", m_name.c_str()); + std::vector<basic_block_t*>::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { + printf("\t"); + std::set<int>::iterator s; + for (s = (*bb_itr)->successor_ids.begin();s != (*bb_itr)->successor_ids.end();s++) { + unsigned succ_bb = *s; + printf("%d -> %d; ", (*bb_itr)->bb_id, succ_bb ); + } + printf("\n"); + } + printf("}\n"); } -unsigned ptx_kernel_shmem_size(void *kernel_impl) { - function_info *f = (function_info *)kernel_impl; - const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); - return kernel_info->smem; +unsigned ptx_kernel_shmem_size( void *kernel_impl ) +{ + function_info *f = (function_info*)kernel_impl; + const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); + return kernel_info->smem; } -unsigned ptx_kernel_nregs(void *kernel_impl) { - function_info *f = (function_info *)kernel_impl; - const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); - return kernel_info->regs; +unsigned ptx_kernel_nregs( void *kernel_impl ) +{ + function_info *f = (function_info*)kernel_impl; + const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); + return kernel_info->regs; } -unsigned type_info_key::type_decode(size_t &size, int &basic_type) const { - int type = scalar_type(); - return type_decode(type, size, basic_type); +unsigned type_info_key::type_decode( size_t &size, int &basic_type ) const +{ + int type = scalar_type(); + return type_decode(type,size,basic_type); } -unsigned type_info_key::type_decode(int type, size_t &size, int &basic_type) { - switch (type) { - case S8_TYPE: - size = 8; - basic_type = 1; - return 0; - case S16_TYPE: - size = 16; - basic_type = 1; - return 1; - case S32_TYPE: - size = 32; - basic_type = 1; - return 2; - case S64_TYPE: - size = 64; - basic_type = 1; - return 3; - case U8_TYPE: - size = 8; - basic_type = 0; - return 4; - case U16_TYPE: - size = 16; - basic_type = 0; - return 5; - case U32_TYPE: - size = 32; - basic_type = 0; - return 6; - case U64_TYPE: - size = 64; - basic_type = 0; - return 7; - case F16_TYPE: - size = 16; - basic_type = -1; - return 8; - case F32_TYPE: - size = 32; - basic_type = -1; - return 9; - case F64_TYPE: - size = 64; - basic_type = -1; - return 10; - case FF64_TYPE: - size = 64; - basic_type = -1; - return 10; - case PRED_TYPE: - size = 1; - basic_type = 2; - return 11; - case B8_TYPE: - size = 8; - basic_type = 0; - return 12; - case B16_TYPE: - size = 16; - basic_type = 0; - return 13; - case B32_TYPE: - size = 32; - basic_type = 0; - return 14; - case B64_TYPE: - size = 64; - basic_type = 0; - return 15; - case BB64_TYPE: - size = 64; - basic_type = 0; - return 15; - case BB128_TYPE: - size = 128; - basic_type = 0; - return 16; - case TEXREF_TYPE: - case SAMPLERREF_TYPE: - case SURFREF_TYPE: - size = 32; - basic_type = 3; - return 16; - default: - printf("ERROR ** type_decode() does not know about \"%s\"\n", - decode_token(type)); - assert(0); +unsigned type_info_key::type_decode( int type, size_t &size, int &basic_type ) +{ + switch ( type ) { + case S8_TYPE: size=8; basic_type=1; return 0; + case S16_TYPE: size=16; basic_type=1; return 1; + case S32_TYPE: size=32; basic_type=1; return 2; + case S64_TYPE: size=64; basic_type=1; return 3; + case U8_TYPE: size=8; basic_type=0; return 4; + case U16_TYPE: size=16; basic_type=0; return 5; + case U32_TYPE: size=32; basic_type=0; return 6; + case U64_TYPE: size=64; basic_type=0; return 7; + case F16_TYPE: size=16; basic_type=-1; return 8; + case F32_TYPE: size=32; basic_type=-1; return 9; + case F64_TYPE: size=64; basic_type=-1; return 10; + case FF64_TYPE: size=64; basic_type=-1; return 10; + case PRED_TYPE: size=1; basic_type=2; return 11; + case B8_TYPE: size=8; basic_type=0; return 12; + case B16_TYPE: size=16; basic_type=0; return 13; + case B32_TYPE: size=32; basic_type=0; return 14; + case B64_TYPE: size=64; basic_type=0; return 15; + case BB64_TYPE: size=64; basic_type=0; return 15; + case BB128_TYPE: size=128; basic_type=0; return 16; + case TEXREF_TYPE: case SAMPLERREF_TYPE: case SURFREF_TYPE: + size=32; basic_type=3; return 16; + default: + printf("ERROR ** type_decode() does not know about \"%s\"\n", decode_token(type) ); + assert(0); return 0xDEADBEEF; - } + } } -arg_buffer_t copy_arg_to_buffer(ptx_thread_info *thread, - operand_info actual_param_op, - const symbol *formal_param) { - if (actual_param_op.is_reg()) { - ptx_reg_t value = thread->get_reg(actual_param_op.get_symbol()); - return arg_buffer_t(formal_param, actual_param_op, value); - } else if (actual_param_op.is_param_local()) { - unsigned size = formal_param->get_size_in_bytes(); - addr_t frame_offset = actual_param_op.get_symbol()->get_address(); - addr_t from_addr = thread->get_local_mem_stack_pointer() + frame_offset; - char buffer[1024]; - assert(size < 1024); - thread->m_local_mem->read(from_addr, size, buffer); - return arg_buffer_t(formal_param, actual_param_op, buffer, size); - } else { - printf( - "GPGPU-Sim PTX: ERROR ** need to add support for this operand type in " - "call/return\n"); - abort(); - } +arg_buffer_t copy_arg_to_buffer(ptx_thread_info * thread, operand_info actual_param_op, const symbol * formal_param) +{ + if( actual_param_op.is_reg() ) { + ptx_reg_t value = thread->get_reg(actual_param_op.get_symbol()); + return arg_buffer_t(formal_param,actual_param_op,value); + } else if ( actual_param_op.is_param_local() ) { + unsigned size=formal_param->get_size_in_bytes(); + addr_t frame_offset = actual_param_op.get_symbol()->get_address(); + addr_t from_addr = thread->get_local_mem_stack_pointer() + frame_offset; + char buffer[1024]; + assert(size<1024); + thread->m_local_mem->read(from_addr,size,buffer); + return arg_buffer_t(formal_param,actual_param_op,buffer,size); + } else { + printf("GPGPU-Sim PTX: ERROR ** need to add support for this operand type in call/return\n"); + abort(); + } } -void copy_args_into_buffer_list(const ptx_instruction *pI, - ptx_thread_info *thread, - const function_info *target_func, - arg_buffer_list_t &arg_values) { - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - for (unsigned arg = 0; arg < n_args; arg++) { - const operand_info &actual_param_op = - pI->operand_lookup(n_return + 1 + arg); - const symbol *formal_param = target_func->get_arg(arg); - arg_values.push_back( - copy_arg_to_buffer(thread, actual_param_op, formal_param)); - } +void copy_args_into_buffer_list( const ptx_instruction * pI, + ptx_thread_info * thread, + const function_info * target_func, + arg_buffer_list_t &arg_values ) +{ + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + for( unsigned arg=0; arg < n_args; arg ++ ) { + const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); + const symbol *formal_param = target_func->get_arg(arg); + arg_values.push_back( copy_arg_to_buffer(thread, actual_param_op, formal_param) ); + } } -void copy_buffer_to_frame(ptx_thread_info *thread, const arg_buffer_t &a) { - if (a.is_reg()) { - ptx_reg_t value = a.get_reg(); - operand_info dst_reg = - operand_info(a.get_dst(), thread->get_gpu()->gpgpu_ctx); - thread->set_reg(dst_reg.get_symbol(), value); - } else { - const void *buffer = a.get_param_buffer(); - size_t size = a.get_param_buffer_size(); - const symbol *dst = a.get_dst(); - addr_t frame_offset = dst->get_address(); - addr_t to_addr = thread->get_local_mem_stack_pointer() + frame_offset; - thread->m_local_mem->write(to_addr, size, buffer, NULL, NULL); - } +void copy_buffer_to_frame(ptx_thread_info * thread, const arg_buffer_t &a) +{ + if( a.is_reg() ) { + ptx_reg_t value = a.get_reg(); + operand_info dst_reg = operand_info(a.get_dst(), thread->get_gpu()->gpgpu_ctx); + thread->set_reg(dst_reg.get_symbol(),value); + } else { + const void *buffer = a.get_param_buffer(); + size_t size = a.get_param_buffer_size(); + const symbol *dst = a.get_dst(); + addr_t frame_offset = dst->get_address(); + addr_t to_addr = thread->get_local_mem_stack_pointer() + frame_offset; + thread->m_local_mem->write(to_addr,size,buffer,NULL,NULL); + } } -void copy_buffer_list_into_frame(ptx_thread_info *thread, - arg_buffer_list_t &arg_values) { - arg_buffer_list_t::iterator a; - for (a = arg_values.begin(); a != arg_values.end(); a++) { - copy_buffer_to_frame(thread, *a); - } +void copy_buffer_list_into_frame(ptx_thread_info * thread, arg_buffer_list_t &arg_values) +{ + arg_buffer_list_t::iterator a; + for( a=arg_values.begin(); a != arg_values.end(); a++ ) { + copy_buffer_to_frame(thread, *a); + } } -static std::list<operand_info> check_operands( - int opcode, const std::list<int> &scalar_type, - const std::list<operand_info> &operands, gpgpu_context *ctx) { - static int g_warn_literal_operands_two_type_inst; - if ((opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || - (opcode == TEX_OP) || (opcode == MMA_OP) || (opcode == DP4A_OP)) { - // just make sure these do not have have const operands... - if (!g_warn_literal_operands_two_type_inst) { - std::list<operand_info>::const_iterator o; - for (o = operands.begin(); o != operands.end(); o++) { - const operand_info &op = *o; - if (op.is_literal()) { - printf( - "GPGPU-Sim PTX: PTX uses two scalar type intruction with literal " - "operand.\n"); - g_warn_literal_operands_two_type_inst = 1; - } - } - } - } else { - assert(scalar_type.size() < 2); - if (scalar_type.size() == 1) { - std::list<operand_info> result; - int inst_type = scalar_type.front(); - std::list<operand_info>::const_iterator o; - for (o = operands.begin(); o != operands.end(); o++) { - const operand_info &op = *o; - if (op.is_literal()) { - if ((op.get_type() == double_op_t) && (inst_type == F32_TYPE)) { - ptx_reg_t v = op.get_literal_value(); - float u = (float)v.f64; - operand_info n(u, ctx); - result.push_back(n); - } else { - result.push_back(op); - } - } else { - result.push_back(op); + + +static std::list<operand_info> check_operands( int opcode, + const std::list<int> &scalar_type, + const std::list<operand_info> &operands, + gpgpu_context* ctx) +{ + static int g_warn_literal_operands_two_type_inst; + if( (opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || (opcode == TEX_OP) || (opcode==MMA_OP) || (opcode == DP4A_OP)) { + // just make sure these do not have have const operands... + if( !g_warn_literal_operands_two_type_inst ) { + std::list<operand_info>::const_iterator o; + for( o = operands.begin(); o != operands.end(); o++ ) { + const operand_info &op = *o; + if( op.is_literal() ) { + printf("GPGPU-Sim PTX: PTX uses two scalar type intruction with literal operand.\n"); + g_warn_literal_operands_two_type_inst = 1; + } + } } - } - return result; + } else { + assert( scalar_type.size() < 2 ); + if( scalar_type.size() == 1 ) { + std::list<operand_info> result; + int inst_type = scalar_type.front(); + std::list<operand_info>::const_iterator o; + for( o = operands.begin(); o != operands.end(); o++ ) { + const operand_info &op = *o; + if( op.is_literal() ) { + if( (op.get_type() == double_op_t) && (inst_type == F32_TYPE) ) { + ptx_reg_t v = op.get_literal_value(); + float u = (float)v.f64; + operand_info n(u, ctx); + result.push_back(n); + } else { + result.push_back(op); + } + } else { + result.push_back(op); + } + } + return result; + } } - } - return operands; + return operands; } -ptx_instruction::ptx_instruction( - int opcode, const symbol *pred, int neg_pred, int pred_mod, symbol *label, - const std::list<operand_info> &operands, const operand_info &return_var, - const std::list<int> &options, const std::list<int> &wmma_options, - const std::list<int> &scalar_type, memory_space_t space_spec, - const char *file, unsigned line, const char *source, - const core_config *config, gpgpu_context *ctx) - : warp_inst_t(config), m_return_var(ctx) { - gpgpu_ctx = ctx; - m_uid = ++(ctx->g_num_ptx_inst_uid); - m_PC = 0; - m_opcode = opcode; - m_pred = pred; - m_neg_pred = neg_pred; - m_pred_mod = pred_mod; - m_label = label; - const std::list<operand_info> checked_operands = - check_operands(opcode, scalar_type, operands, ctx); - m_operands.insert(m_operands.begin(), checked_operands.begin(), - checked_operands.end()); - m_return_var = return_var; - m_options = options; - m_wmma_options = wmma_options; - m_wide = false; - m_hi = false; - m_lo = false; - m_uni = false; - m_exit = false; - m_abs = false; - m_neg = false; - m_to_option = false; - m_cache_option = 0; - m_rounding_mode = RN_OPTION; - m_compare_op = -1; - m_saturation_mode = 0; - m_geom_spec = 0; - m_vector_spec = 0; - m_atomic_spec = 0; - m_membar_level = 0; - m_inst_size = 8; // bytes - int rr = 0; - std::list<int>::const_iterator i; - unsigned n = 1; - for (i = wmma_options.begin(); i != wmma_options.end(); i++, n++) { - int last_ptx_inst_option = *i; - switch (last_ptx_inst_option) { - case SYNC_OPTION: - case LOAD_A: - case LOAD_B: - case LOAD_C: - case STORE_D: - case MMA: - m_wmma_type = last_ptx_inst_option; - break; - case ROW: - case COL: - m_wmma_layout[rr++] = last_ptx_inst_option; - break; - case M16N16K16: - case M32N8K16: - case M8N32K16: - break; - default: - assert(0); - break; - } - } - rr = 0; - n = 1; - for (i = options.begin(); i != options.end(); i++, n++) { - int last_ptx_inst_option = *i; - switch (last_ptx_inst_option) { + +ptx_instruction::ptx_instruction( int opcode, + const symbol *pred, + int neg_pred, + int pred_mod, + symbol *label, + const std::list<operand_info> &operands, + const operand_info &return_var, + const std::list<int> &options, + const std::list<int> &wmma_options, + const std::list<int> &scalar_type, + memory_space_t space_spec, + const char *file, + unsigned line, + const char *source, + const core_config *config, + gpgpu_context* ctx ) : warp_inst_t(config), m_return_var(ctx) +{ + gpgpu_ctx = ctx; + m_uid = ++(ctx->g_num_ptx_inst_uid); + m_PC = 0; + m_opcode = opcode; + m_pred = pred; + m_neg_pred = neg_pred; + m_pred_mod = pred_mod; + m_label = label; + const std::list<operand_info> checked_operands = check_operands(opcode,scalar_type,operands, ctx); + m_operands.insert(m_operands.begin(), checked_operands.begin(), checked_operands.end() ); + m_return_var = return_var; + m_options = options; + m_wmma_options = wmma_options; + m_wide = false; + m_hi = false; + m_lo = false; + m_uni = false; + m_exit = false; + m_abs = false; + m_neg = false; + m_to_option = false; + m_cache_option = 0; + m_rounding_mode = RN_OPTION; + m_compare_op = -1; + m_saturation_mode = 0; + m_geom_spec = 0; + m_vector_spec = 0; + m_atomic_spec = 0; + m_membar_level = 0; + m_inst_size = 8; // bytes + int rr=0; + std::list<int>::const_iterator i; + unsigned n=1; + for ( i=wmma_options.begin(); i!= wmma_options.end(); i++, n++ ) { + int last_ptx_inst_option = *i; + switch ( last_ptx_inst_option ) { + case SYNC_OPTION: + case LOAD_A: + case LOAD_B: + case LOAD_C: + case STORE_D: + case MMA: + m_wmma_type=last_ptx_inst_option; + break; + case ROW: + case COL: + m_wmma_layout[rr++]=last_ptx_inst_option; + break; + case M16N16K16: + case M32N8K16: + case M8N32K16: + break; + default: + assert(0); + break; + } + } + rr=0; + n=1; + for ( i=options.begin(); i!= options.end(); i++, n++ ) { + int last_ptx_inst_option = *i; + switch ( last_ptx_inst_option ) { case SYNC_OPTION: case ARRIVE_OPTION: case RED_OPTION: - m_barrier_op = last_ptx_inst_option; - break; + m_barrier_op = last_ptx_inst_option; + break; case EQU_OPTION: case NEU_OPTION: case LTU_OPTION: @@ -1280,16 +1186,16 @@ ptx_instruction::ptx_instruction( case GE_OPTION: case LS_OPTION: case HS_OPTION: - m_compare_op = last_ptx_inst_option; - break; + m_compare_op = last_ptx_inst_option; + break; case NUM_OPTION: case NAN_OPTION: - m_compare_op = last_ptx_inst_option; + m_compare_op = last_ptx_inst_option; // assert(0); // finish this - break; + break; case SAT_OPTION: - m_saturation_mode = 1; - break; + m_saturation_mode = 1; + break; case RNI_OPTION: case RZI_OPTION: case RMI_OPTION: @@ -1298,39 +1204,38 @@ ptx_instruction::ptx_instruction( case RZ_OPTION: case RM_OPTION: case RP_OPTION: - m_rounding_mode = last_ptx_inst_option; - break; + m_rounding_mode = last_ptx_inst_option; + break; case HI_OPTION: - m_compare_op = last_ptx_inst_option; - m_hi = true; - assert(!m_lo); - assert(!m_wide); - break; + m_compare_op = last_ptx_inst_option; + m_hi = true; + assert( !m_lo ); + assert( !m_wide ); + break; case LO_OPTION: - m_compare_op = last_ptx_inst_option; - m_lo = true; - assert(!m_hi); - assert(!m_wide); - break; + m_compare_op = last_ptx_inst_option; + m_lo = true; + assert( !m_hi ); + assert( !m_wide ); + break; case WIDE_OPTION: - m_wide = true; - assert(!m_lo); - assert(!m_hi); - break; + m_wide = true; + assert( !m_lo ); + assert( !m_hi ); + break; case UNI_OPTION: - m_uni = true; // don't care... < now we DO care when constructing - // flowgraph> - break; + m_uni = true; // don't care... < now we DO care when constructing flowgraph> + break; case GEOM_MODIFIER_1D: case GEOM_MODIFIER_2D: case GEOM_MODIFIER_3D: - m_geom_spec = last_ptx_inst_option; - break; + m_geom_spec = last_ptx_inst_option; + break; case V2_TYPE: case V3_TYPE: case V4_TYPE: - m_vector_spec = last_ptx_inst_option; - break; + m_vector_spec = last_ptx_inst_option; + break; case ATOMIC_AND: case ATOMIC_OR: case ATOMIC_XOR: @@ -1341,225 +1246,223 @@ ptx_instruction::ptx_instruction( case ATOMIC_DEC: case ATOMIC_MIN: case ATOMIC_MAX: - m_atomic_spec = last_ptx_inst_option; - break; + m_atomic_spec = last_ptx_inst_option; + break; case APPROX_OPTION: - break; + break; case FULL_OPTION: - break; + break; case ANY_OPTION: - m_vote_mode = vote_any; - break; + m_vote_mode = vote_any; + break; case ALL_OPTION: - m_vote_mode = vote_all; - break; + m_vote_mode = vote_all; + break; case BALLOT_OPTION: - m_vote_mode = vote_ballot; - break; + m_vote_mode = vote_ballot; + break; case GLOBAL_OPTION: - m_membar_level = GLOBAL_OPTION; - break; + m_membar_level = GLOBAL_OPTION; + break; case CTA_OPTION: - m_membar_level = CTA_OPTION; - break; + m_membar_level = CTA_OPTION; + break; case SYS_OPTION: - m_membar_level = SYS_OPTION; - break; + m_membar_level = SYS_OPTION; + break; case FTZ_OPTION: - break; + break; case EXIT_OPTION: - m_exit = true; - break; + m_exit = true; + break; case ABS_OPTION: - m_abs = true; - break; + m_abs = true; + break; case NEG_OPTION: - m_neg = true; - break; + m_neg = true; + break; case TO_OPTION: - m_to_option = true; - break; - case CA_OPTION: - case CG_OPTION: - case CS_OPTION: - case LU_OPTION: - case CV_OPTION: - m_cache_option = last_ptx_inst_option; - break; + m_to_option = true; + break; + case CA_OPTION: case CG_OPTION: case CS_OPTION: case LU_OPTION: case CV_OPTION: + m_cache_option = last_ptx_inst_option; + break; case HALF_OPTION: - m_inst_size = 4; // bytes - break; + m_inst_size = 4; // bytes + break; case EXTP_OPTION: - break; + break; case NC_OPTION: - m_cache_option = last_ptx_inst_option; - break; + m_cache_option = last_ptx_inst_option; + break; case UP_OPTION: case DOWN_OPTION: case BFLY_OPTION: case IDX_OPTION: - m_shfl_op = last_ptx_inst_option; - break; + m_shfl_op = last_ptx_inst_option; + break; case PRMT_F4E_MODE: case PRMT_B4E_MODE: case PRMT_RC8_MODE: case PRMT_ECL_MODE: case PRMT_ECR_MODE: case PRMT_RC16_MODE: - m_prmt_op = last_ptx_inst_option; - break; + m_prmt_op = last_ptx_inst_option; + break; default: - assert(0); - break; - } - } - m_scalar_type = scalar_type; - m_space_spec = space_spec; - if ((opcode == ST_OP || opcode == LD_OP || opcode == LDU_OP) && - (space_spec == undefined_space)) { - m_space_spec = generic_space; - } - for (std::vector<operand_info>::const_iterator i = m_operands.begin(); - i != m_operands.end(); ++i) { - const operand_info &op = *i; - if (op.get_addr_space() != undefined_space) - m_space_spec = op.get_addr_space(); // TODO: can have more than one - // memory space for ptxplus (g8x) - // inst - } - if (opcode == TEX_OP) m_space_spec = tex_space; + assert(0); + break; + } + } + m_scalar_type = scalar_type; + m_space_spec = space_spec; + if( ( opcode == ST_OP || opcode == LD_OP || opcode == LDU_OP ) && (space_spec == undefined_space) ) { + m_space_spec = generic_space; + } + for( std::vector<operand_info>::const_iterator i=m_operands.begin(); i!=m_operands.end(); ++i) { + const operand_info &op = *i; + if( op.get_addr_space() != undefined_space ) + m_space_spec = op.get_addr_space(); // TODO: can have more than one memory space for ptxplus (g8x) inst + } + if( opcode == TEX_OP ) + m_space_spec = tex_space; + + m_source_file = file?file:"<unknown>"; + m_source_line = line; + m_source = source; + // Trim tabs + m_source.erase( std::remove( m_source.begin(), m_source.end(), '\t' ), m_source.end() ); - m_source_file = file ? file : "<unknown>"; - m_source_line = line; - m_source = source; - // Trim tabs - m_source.erase(std::remove(m_source.begin(), m_source.end(), '\t'), - m_source.end()); + if (opcode == CALL_OP) { + const operand_info &target = func_addr(); + assert( target.is_function_address() ); + const symbol *func_addr = target.get_symbol(); + const function_info *target_func = func_addr->get_pc(); + std::string fname = target_func->get_name(); - if (opcode == CALL_OP) { - const operand_info &target = func_addr(); - assert(target.is_function_address()); - const symbol *func_addr = target.get_symbol(); - const function_info *target_func = func_addr->get_pc(); - std::string fname = target_func->get_name(); + if (fname =="vprintf"){ + m_is_printf = true; + } + if(fname == "cudaStreamCreateWithFlags") + m_is_cdp = 1; + if(fname == "cudaGetParameterBufferV2") + m_is_cdp = 2; + if(fname == "cudaLaunchDeviceV2") + m_is_cdp = 4; - if (fname == "vprintf") { - m_is_printf = true; - } - if (fname == "cudaStreamCreateWithFlags") m_is_cdp = 1; - if (fname == "cudaGetParameterBufferV2") m_is_cdp = 2; - if (fname == "cudaLaunchDeviceV2") m_is_cdp = 4; - } + } } -void ptx_instruction::print_insn() const { - print_insn(stdout); - fflush(stdout); +void ptx_instruction::print_insn() const +{ + print_insn(stdout); + fflush(stdout); } -void ptx_instruction::print_insn(FILE *fp) const { - fprintf(fp, "%s", to_string().c_str()); +void ptx_instruction::print_insn( FILE *fp ) const +{ + fprintf( fp, "%s", to_string().c_str() ); } -std::string ptx_instruction::to_string() const { - char buf[STR_SIZE]; - unsigned used_bytes = 0; - if (!is_label()) { - used_bytes += - snprintf(buf + used_bytes, STR_SIZE - used_bytes, " PC=0x%03x ", m_PC); - } else { - used_bytes += - snprintf(buf + used_bytes, STR_SIZE - used_bytes, " "); - } - used_bytes += - snprintf(buf + used_bytes, STR_SIZE - used_bytes, "(%s:%d) %s", - m_source_file.c_str(), m_source_line, m_source.c_str()); - return std::string(buf); +std::string ptx_instruction::to_string() const +{ + char buf[ STR_SIZE ]; + unsigned used_bytes = 0; + if( !is_label() ) { + used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, " PC=0x%03x ", m_PC ); + } else { + used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, " " ); + } + used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, + "(%s:%d) %s", + m_source_file.c_str(), m_source_line, + m_source.c_str() ); + return std::string( buf ); } -operand_info ptx_instruction::get_pred() const { - return operand_info(m_pred, gpgpu_ctx); +operand_info ptx_instruction::get_pred() const +{ + return operand_info( m_pred, gpgpu_ctx); } -function_info::function_info(int entry_point, gpgpu_context *ctx) { - gpgpu_ctx = ctx; - m_uid = (gpgpu_ctx->function_info_sm_next_uid)++; - m_entry_point = (entry_point == 1) ? true : false; - m_extern = (entry_point == 2) ? true : false; - num_reconvergence_pairs = 0; - m_symtab = NULL; - m_assembled = false; - m_return_var_sym = NULL; - m_kernel_info.cmem = 0; - m_kernel_info.lmem = 0; - m_kernel_info.regs = 0; - m_kernel_info.smem = 0; - m_local_mem_framesize = 0; - m_args_aligned_size = -1; - pdom_done = false; // initialize it to false + +function_info::function_info(int entry_point, gpgpu_context* ctx ) +{ + gpgpu_ctx = ctx; + m_uid = (gpgpu_ctx->function_info_sm_next_uid)++; + m_entry_point = (entry_point==1)?true:false; + m_extern = (entry_point==2)?true:false; + num_reconvergence_pairs = 0; + m_symtab = NULL; + m_assembled = false; + m_return_var_sym = NULL; + m_kernel_info.cmem = 0; + m_kernel_info.lmem = 0; + m_kernel_info.regs = 0; + m_kernel_info.smem = 0; + m_local_mem_framesize = 0; + m_args_aligned_size = -1; + pdom_done = false; //initialize it to false } -unsigned function_info::print_insn(unsigned pc, FILE *fp) const { - unsigned inst_size = 1; // return offset to next instruction or 1 if unknown - unsigned index = pc - m_start_PC; - char command[1024]; - char buffer[1024]; - memset(command, 0, 1024); - memset(buffer, 0, 1024); - snprintf(command, 1024, "c++filt -p %s", m_name.c_str()); - FILE *p = popen(command, "r"); - buffer[0] = 0; - assert(fgets(buffer, 1023, p) != NULL); - // Remove trailing "\n" in buffer - char *c; - if ((c = strchr(buffer, '\n')) != NULL) *c = '\0'; - fprintf(fp, "%s", buffer); - if (index >= m_instr_mem_size) { - fprintf(fp, "<past last instruction (max pc=%u)>", - m_start_PC + m_instr_mem_size - 1); - } else { - if (m_instr_mem[index] != NULL) { - m_instr_mem[index]->print_insn(fp); - inst_size = m_instr_mem[index]->isize; - } else - fprintf(fp, "<no instruction at pc = %u>", pc); - } - pclose(p); - return inst_size; +unsigned function_info::print_insn( unsigned pc, FILE * fp ) const +{ + unsigned inst_size=1; // return offset to next instruction or 1 if unknown + unsigned index = pc - m_start_PC; + char command[1024]; + char buffer[1024]; + memset(command, 0, 1024); + memset(buffer, 0, 1024); + snprintf(command,1024,"c++filt -p %s",m_name.c_str()); + FILE *p = popen(command,"r"); + buffer[0]=0; + assert(fgets(buffer, 1023, p) != NULL); + // Remove trailing "\n" in buffer + char *c; + if ((c=strchr(buffer, '\n')) != NULL) *c = '\0'; + fprintf(fp,"%s",buffer); + if ( index >= m_instr_mem_size ) { + fprintf(fp, "<past last instruction (max pc=%u)>", m_start_PC + m_instr_mem_size - 1 ); + } else { + if ( m_instr_mem[index] != NULL ) { + m_instr_mem[index]->print_insn(fp); + inst_size = m_instr_mem[index]->isize; + } else + fprintf(fp, "<no instruction at pc = %u>", pc ); + } + pclose(p); + return inst_size; } -std::string function_info::get_insn_str(unsigned pc) const { - unsigned index = pc - m_start_PC; - if (index >= m_instr_mem_size) { - char buff[STR_SIZE]; - buff[STR_SIZE - 1] = '\0'; - snprintf(buff, STR_SIZE, "<past last instruction (max pc=%u)>", - m_start_PC + m_instr_mem_size - 1); - return std::string(buff); - } else { - if (m_instr_mem[index] != NULL) { - return m_instr_mem[index]->to_string(); - } else { +std::string function_info::get_insn_str( unsigned pc ) const +{ + unsigned index = pc - m_start_PC; + if ( index >= m_instr_mem_size ) { char buff[STR_SIZE]; - buff[STR_SIZE - 1] = '\0'; - snprintf(buff, STR_SIZE, "<no instruction at pc = %u>", pc); + buff[STR_SIZE-1] = '\0'; + snprintf(buff, STR_SIZE, "<past last instruction (max pc=%u)>", m_start_PC + m_instr_mem_size - 1 ); return std::string(buff); - } - } + } else { + if ( m_instr_mem[index] != NULL ) { + return m_instr_mem[index]->to_string(); + } else { + char buff[STR_SIZE]; + buff[STR_SIZE-1] = '\0'; + snprintf(buff, STR_SIZE, "<no instruction at pc = %u>", pc ); + return std::string(buff); + } + } } -void gpgpu_ptx_assemble(std::string kname, void *kinfo) { - function_info *func_info = (function_info *)kinfo; - if ((function_info *)kinfo == NULL) { - printf("GPGPU-Sim PTX: Warning - missing function definition \'%s\'\n", - kname.c_str()); - return; - } - if (func_info->is_extern()) { - printf( - "GPGPU-Sim PTX: skipping assembly for extern declared function " - "\'%s\'\n", - func_info->get_name().c_str()); - return; - } - func_info->ptx_assemble(); +void gpgpu_ptx_assemble( std::string kname, void *kinfo ) +{ + function_info *func_info = (function_info *)kinfo; + if((function_info *)kinfo == NULL) { + printf("GPGPU-Sim PTX: Warning - missing function definition \'%s\'\n", kname.c_str()); + return; + } + if( func_info->is_extern() ) { + printf("GPGPU-Sim PTX: skipping assembly for extern declared function \'%s\'\n", func_info->get_name().c_str() ); + return; + } + func_info->ptx_assemble(); } diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 6d82374..f4c5c37 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -32,13 +30,13 @@ #include "../abstract_hardware_model.h" -#include <assert.h> #include <cstdlib> #include <cstring> +#include <string> #include <list> #include <map> -#include <string> #include <vector> +#include <assert.h> //#include "ptx.tab.h" #include "ptx_sim.h" @@ -48,1518 +46,1537 @@ class gpgpu_context; class type_info_key { - public: - type_info_key() { - m_is_non_arch_reg = false; - m_init = false; - } - type_info_key(memory_space_t space_spec, int scalar_type_spec, - int vector_spec, int alignment_spec, int extern_spec, - int array_dim) { - m_is_non_arch_reg = false; - m_init = true; - m_space_spec = space_spec; - m_scalar_type_spec = scalar_type_spec; - m_vector_spec = vector_spec; - m_alignment_spec = alignment_spec; - m_extern_spec = extern_spec; - m_array_dim = array_dim; - m_is_function = 0; - } - void set_is_func() { - assert(!m_init); - m_init = true; - m_space_spec = undefined_space; - m_scalar_type_spec = 0; - m_vector_spec = 0; - m_alignment_spec = 0; - m_extern_spec = 0; - m_array_dim = 0; - m_is_function = 1; - } - - void set_array_dim(int array_dim) { m_array_dim = array_dim; } - int get_array_dim() const { - assert(m_init); - return m_array_dim; - } - void set_is_non_arch_reg() { m_is_non_arch_reg = true; } +public: + type_info_key() + { + m_is_non_arch_reg = false; + m_init = false; + } + type_info_key( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec, int array_dim ) + { + m_is_non_arch_reg = false; + m_init = true; + m_space_spec = space_spec; + m_scalar_type_spec = scalar_type_spec; + m_vector_spec = vector_spec; + m_alignment_spec = alignment_spec; + m_extern_spec = extern_spec; + m_array_dim = array_dim; + m_is_function = 0; + } + void set_is_func() + { + assert(!m_init); + m_init = true; + m_space_spec = undefined_space; + m_scalar_type_spec = 0; + m_vector_spec = 0; + m_alignment_spec = 0; + m_extern_spec = 0; + m_array_dim = 0; + m_is_function = 1; + } - bool is_non_arch_reg() const { return m_is_non_arch_reg; } - bool is_reg() const { return m_space_spec == reg_space; } - bool is_param_kernel() const { return m_space_spec == param_space_kernel; } - bool is_param_local() const { return m_space_spec == param_space_local; } - bool is_param_unclassified() const { - return m_space_spec == param_space_unclassified; - } - bool is_global() const { return m_space_spec == global_space; } - bool is_local() const { return m_space_spec == local_space; } - bool is_shared() const { return m_space_spec == shared_space; } - bool is_const() const { return m_space_spec.get_type() == const_space; } - bool is_tex() const { return m_space_spec == tex_space; } - bool is_func_addr() const { return m_is_function ? true : false; } - int scalar_type() const { return m_scalar_type_spec; } - int get_alignment_spec() const { return m_alignment_spec; } - unsigned type_decode(size_t &size, int &t) const; - static unsigned type_decode(int type, size_t &size, int &t); - memory_space_t get_memory_space() const { return m_space_spec; } + void set_array_dim( int array_dim ) { m_array_dim = array_dim; } + int get_array_dim() const { assert(m_init); return m_array_dim; } + void set_is_non_arch_reg() { m_is_non_arch_reg = true; } - private: - bool m_init; - memory_space_t m_space_spec; - int m_scalar_type_spec; - int m_vector_spec; - int m_alignment_spec; - int m_extern_spec; - int m_array_dim; - int m_is_function; - bool m_is_non_arch_reg; + bool is_non_arch_reg() const { return m_is_non_arch_reg; } + bool is_reg() const { return m_space_spec == reg_space;} + bool is_param_kernel() const { return m_space_spec == param_space_kernel;} + bool is_param_local() const { return m_space_spec == param_space_local; } + bool is_param_unclassified() const { return m_space_spec == param_space_unclassified; } + bool is_global() const { return m_space_spec == global_space;} + bool is_local() const { return m_space_spec == local_space;} + bool is_shared() const { return m_space_spec == shared_space;} + bool is_const() const { return m_space_spec.get_type() == const_space;} + bool is_tex() const { return m_space_spec == tex_space;} + bool is_func_addr() const { return m_is_function?true:false; } + int scalar_type() const { return m_scalar_type_spec;} + int get_alignment_spec() const { return m_alignment_spec;} + unsigned type_decode( size_t &size, int &t ) const; + static unsigned type_decode( int type, size_t &size, int &t ); + memory_space_t get_memory_space() const { return m_space_spec; } +private: + bool m_init; + memory_space_t m_space_spec; + int m_scalar_type_spec; + int m_vector_spec; + int m_alignment_spec; + int m_extern_spec; + int m_array_dim; + int m_is_function; + bool m_is_non_arch_reg; - friend struct type_info_key_compare; + friend struct type_info_key_compare; }; class symbol_table; struct type_info_key_compare { - bool operator()(const type_info_key &a, const type_info_key &b) const { - assert(a.m_init && b.m_init); - if (a.m_space_spec < b.m_space_spec) return true; - if (a.m_scalar_type_spec < b.m_scalar_type_spec) return true; - if (a.m_vector_spec < b.m_vector_spec) return true; - if (a.m_alignment_spec < b.m_alignment_spec) return true; - if (a.m_extern_spec < b.m_extern_spec) return true; - if (a.m_array_dim < b.m_array_dim) return true; - if (a.m_is_function < b.m_is_function) return true; + bool operator()( const type_info_key &a, const type_info_key &b ) const + { + assert( a.m_init && b.m_init ); + if ( a.m_space_spec < b.m_space_spec ) return true; + if ( a.m_scalar_type_spec < b.m_scalar_type_spec ) return true; + if ( a.m_vector_spec < b.m_vector_spec ) return true; + if ( a.m_alignment_spec < b.m_alignment_spec ) return true; + if ( a.m_extern_spec < b.m_extern_spec ) return true; + if ( a.m_array_dim < b.m_array_dim ) return true; + if ( a.m_is_function < b.m_is_function ) return true; - return false; - } + return false; + } }; class type_info { - public: - type_info(symbol_table *scope, type_info_key t) { m_type_info = t; } - const type_info_key &get_key() const { return m_type_info; } +public: + type_info( symbol_table *scope, type_info_key t ) + { + m_type_info = t; + } + const type_info_key &get_key() const { return m_type_info;} - private: - symbol_table *m_scope; - type_info_key m_type_info; +private: + symbol_table *m_scope; + type_info_key m_type_info; }; enum operand_type { - reg_t, - vector_t, - builtin_t, - address_t, - memory_t, - float_op_t, - double_op_t, - int_t, - unsigned_t, - symbolic_t, - label_t, - v_reg_t, - v_float_op_t, - v_double_op_t, - v_int_t, - v_unsigned_t, - undef_t + reg_t, vector_t, builtin_t, address_t, memory_t, float_op_t, double_op_t, int_t, + unsigned_t, symbolic_t, label_t, v_reg_t, v_float_op_t, v_double_op_t, + v_int_t, v_unsigned_t, undef_t }; class operand_info; class symbol { - public: - symbol(const char *name, const type_info *type, const char *location, - unsigned size, gpgpu_context *ctx) { - gpgpu_ctx = ctx; - m_uid = get_uid(); - m_name = name; - m_decl_location = location; - m_type = type; - m_size = size; - m_address_valid = false; - m_is_label = false; - m_is_shared = false; - m_is_const = false; - m_is_global = false; - m_is_local = false; - m_is_param_local = false; - m_is_param_kernel = false; - m_is_tex = false; - m_is_func_addr = false; - m_reg_num_valid = false; - m_function = NULL; - m_reg_num = (unsigned)-1; - m_arch_reg_num = (unsigned)-1; - m_address = (unsigned)-1; - m_initializer.clear(); - if (type) m_is_shared = type->get_key().is_shared(); - if (type) m_is_const = type->get_key().is_const(); - if (type) m_is_global = type->get_key().is_global(); - if (type) m_is_local = type->get_key().is_local(); - if (type) m_is_param_local = type->get_key().is_param_local(); - if (type) m_is_param_kernel = type->get_key().is_param_kernel(); - if (type) m_is_tex = type->get_key().is_tex(); - if (type) m_is_func_addr = type->get_key().is_func_addr(); - } - unsigned get_size_in_bytes() const { return m_size; } - const std::string &name() const { return m_name; } - const std::string &decl_location() const { return m_decl_location; } - const type_info *type() const { return m_type; } - addr_t get_address() const { - assert(m_is_label || - !m_type->get_key().is_reg()); // todo : other assertions - assert(m_address_valid); - return m_address; - } - function_info *get_pc() const { return m_function; } - void set_regno(unsigned regno, unsigned arch_regno) { - m_reg_num_valid = true; - m_reg_num = regno; - m_arch_reg_num = arch_regno; - } +public: + symbol( const char *name, const type_info *type, const char *location, unsigned size, gpgpu_context* ctx ) + { + gpgpu_ctx = ctx; + m_uid = get_uid(); + m_name = name; + m_decl_location = location; + m_type = type; + m_size = size; + m_address_valid = false; + m_is_label = false; + m_is_shared = false; + m_is_const = false; + m_is_global = false; + m_is_local = false; + m_is_param_local = false; + m_is_param_kernel = false; + m_is_tex = false; + m_is_func_addr = false; + m_reg_num_valid = false; + m_function = NULL; + m_reg_num=(unsigned)-1; + m_arch_reg_num=(unsigned)-1; + m_address=(unsigned)-1; + m_initializer.clear(); + if ( type ) m_is_shared = type->get_key().is_shared(); + if ( type ) m_is_const = type->get_key().is_const(); + if ( type ) m_is_global = type->get_key().is_global(); + if ( type ) m_is_local = type->get_key().is_local(); + if ( type ) m_is_param_local = type->get_key().is_param_local(); + if ( type ) m_is_param_kernel = type->get_key().is_param_kernel(); + if ( type ) m_is_tex = type->get_key().is_tex(); + if ( type ) m_is_func_addr = type->get_key().is_func_addr(); + } + unsigned get_size_in_bytes() const + { + return m_size; + } + const std::string &name() const { return m_name;} + const std::string &decl_location() const { return m_decl_location;} + const type_info *type() const { return m_type;} + addr_t get_address() const + { + assert( m_is_label || !m_type->get_key().is_reg() ); // todo : other assertions + assert( m_address_valid ); + return m_address; + } + function_info *get_pc() const + { + return m_function; + } + void set_regno( unsigned regno, unsigned arch_regno ) + { + m_reg_num_valid = true; + m_reg_num = regno; + m_arch_reg_num = arch_regno; + } - void set_address(addr_t addr) { - m_address_valid = true; - m_address = addr; - } - void set_label_address(addr_t addr) { - m_address_valid = true; - m_address = addr; - m_is_label = true; - } - void set_function(function_info *func) { - m_function = func; - m_is_func_addr = true; - } + void set_address( addr_t addr ) + { + m_address_valid = true; + m_address = addr; + } + void set_label_address( addr_t addr) + { + m_address_valid = true; + m_address = addr; + m_is_label = true; + } + void set_function( function_info *func ) + { + m_function = func; + m_is_func_addr = true; + } - bool is_label() const { return m_is_label; } - bool is_shared() const { return m_is_shared; } - bool is_sstarr() const { return m_is_sstarr; } - bool is_const() const { return m_is_const; } - bool is_global() const { return m_is_global; } - bool is_local() const { return m_is_local; } - bool is_param_local() const { return m_is_param_local; } - bool is_param_kernel() const { return m_is_param_kernel; } - bool is_tex() const { return m_is_tex; } - bool is_func_addr() const { return m_is_func_addr; } - bool is_reg() const { - if (m_type == NULL) { - return false; - } - return m_type->get_key().is_reg(); - } - bool is_non_arch_reg() const { - if (m_type == NULL) { - return false; - } - return m_type->get_key().is_non_arch_reg(); - } + bool is_label() const { return m_is_label;} + bool is_shared() const { return m_is_shared;} + bool is_sstarr() const { return m_is_sstarr;} + bool is_const() const { return m_is_const;} + bool is_global() const { return m_is_global;} + bool is_local() const { return m_is_local;} + bool is_param_local() const { return m_is_param_local; } + bool is_param_kernel() const { return m_is_param_kernel; } + bool is_tex() const { return m_is_tex;} + bool is_func_addr() const { return m_is_func_addr; } + bool is_reg() const + { + if ( m_type == NULL ) { + return false; + } + return m_type->get_key().is_reg(); + } + bool is_non_arch_reg() const + { + if ( m_type == NULL ) { + return false; + } + return m_type->get_key().is_non_arch_reg(); + } - void add_initializer(const std::list<operand_info> &init); - bool has_initializer() const { return m_initializer.size() > 0; } - std::list<operand_info> get_initializer() const { return m_initializer; } - unsigned reg_num() const { - assert(m_reg_num_valid); - return m_reg_num; - } - unsigned arch_reg_num() const { - assert(m_reg_num_valid); - return m_arch_reg_num; - } - void print_info(FILE *fp) const; - unsigned uid() const { return m_uid; } + void add_initializer( const std::list<operand_info> &init ); + bool has_initializer() const + { + return m_initializer.size() > 0; + } + std::list<operand_info> get_initializer() const + { + return m_initializer; + } + unsigned reg_num() const + { + assert( m_reg_num_valid ); + return m_reg_num; + } + unsigned arch_reg_num() const + { + assert( m_reg_num_valid ); + return m_arch_reg_num; + } + void print_info(FILE *fp) const; + unsigned uid() const { return m_uid; } - private: - gpgpu_context *gpgpu_ctx; - unsigned get_uid(); - unsigned m_uid; - const type_info *m_type; - unsigned m_size; // in bytes - std::string m_name; - std::string m_decl_location; +private: + gpgpu_context* gpgpu_ctx; + unsigned get_uid(); + unsigned m_uid; + const type_info *m_type; + unsigned m_size; // in bytes + std::string m_name; + std::string m_decl_location; - unsigned m_address; - function_info *m_function; // used for function symbols + unsigned m_address; + function_info *m_function; // used for function symbols - bool m_address_valid; - bool m_is_label; - bool m_is_shared; - bool m_is_sstarr; - bool m_is_const; - bool m_is_global; - bool m_is_local; - bool m_is_param_local; - bool m_is_param_kernel; - bool m_is_tex; - bool m_is_func_addr; - unsigned m_reg_num; - unsigned m_arch_reg_num; - bool m_reg_num_valid; + bool m_address_valid; + bool m_is_label; + bool m_is_shared; + bool m_is_sstarr; + bool m_is_const; + bool m_is_global; + bool m_is_local; + bool m_is_param_local; + bool m_is_param_kernel; + bool m_is_tex; + bool m_is_func_addr; + unsigned m_reg_num; + unsigned m_arch_reg_num; + bool m_reg_num_valid; - std::list<operand_info> m_initializer; + std::list<operand_info> m_initializer; }; class symbol_table { - public: - symbol_table(); - symbol_table(const char *scope_name, unsigned entry_point, - symbol_table *parent, gpgpu_context *ctx); - void set_name(const char *name); - const ptx_version &get_ptx_version() const; - unsigned get_sm_target() const; - void set_ptx_version(float ver, unsigned ext); - void set_sm_target(const char *target, const char *ext, const char *ext2); - symbol *lookup(const char *identifier); - std::string get_scope_name() const { return m_scope_name; } - symbol *add_variable(const char *identifier, const type_info *type, - unsigned size, const char *filename, unsigned line); - void add_function(function_info *func, const char *filename, - unsigned linenumber); - bool add_function_decl(const char *name, int entry_point, - function_info **func_info, - symbol_table **symbol_table); - function_info *lookup_function(std::string name); - type_info *add_type(memory_space_t space_spec, int scalar_type_spec, - int vector_spec, int alignment_spec, int extern_spec); - type_info *add_type(function_info *func); - type_info *get_array_type(type_info *base_type, unsigned array_dim); - void set_label_address(const symbol *label, unsigned addr); - unsigned next_reg_num() { return ++m_reg_allocator; } - addr_t get_shared_next() { return m_shared_next; } - addr_t get_sstarr_next() { return m_sstarr_next; } - addr_t get_global_next() { return m_global_next; } - addr_t get_local_next() { return m_local_next; } - addr_t get_tex_next() { return m_tex_next; } - void alloc_shared(unsigned num_bytes) { m_shared_next += num_bytes; } - void alloc_sstarr(unsigned num_bytes) { m_sstarr_next += num_bytes; } - void alloc_global(unsigned num_bytes) { m_global_next += num_bytes; } - void alloc_local(unsigned num_bytes) { m_local_next += num_bytes; } - void alloc_tex(unsigned num_bytes) { m_tex_next += num_bytes; } - - typedef std::list<symbol *>::iterator iterator; - - iterator global_iterator_begin() { return m_globals.begin(); } - iterator global_iterator_end() { return m_globals.end(); } +public: + symbol_table(); + symbol_table( const char *scope_name, unsigned entry_point, symbol_table *parent, gpgpu_context* ctx); + void set_name( const char *name ); + const ptx_version &get_ptx_version() const; + unsigned get_sm_target() const; + void set_ptx_version( float ver, unsigned ext ); + void set_sm_target( const char *target, const char *ext, const char *ext2 ); + symbol* lookup( const char *identifier ); + std::string get_scope_name() const { return m_scope_name; } + symbol *add_variable( const char *identifier, const type_info *type, unsigned size, const char *filename, unsigned line ); + void add_function( function_info *func, const char *filename, unsigned linenumber ); + bool add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **symbol_table ); + function_info *lookup_function(std::string name); + type_info *add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ); + type_info *add_type( function_info *func ); + type_info *get_array_type( type_info *base_type, unsigned array_dim ); + void set_label_address( const symbol *label, unsigned addr ); + unsigned next_reg_num() { return ++m_reg_allocator;} + addr_t get_shared_next() { return m_shared_next;} + addr_t get_sstarr_next() { return m_sstarr_next;} + addr_t get_global_next() { return m_global_next;} + addr_t get_local_next() { return m_local_next;} + addr_t get_tex_next() { return m_tex_next;} + void alloc_shared( unsigned num_bytes ) { m_shared_next += num_bytes;} + void alloc_sstarr( unsigned num_bytes ) { m_sstarr_next += num_bytes;} + void alloc_global( unsigned num_bytes ) { m_global_next += num_bytes;} + void alloc_local( unsigned num_bytes ) { m_local_next += num_bytes;} + void alloc_tex( unsigned num_bytes ) { m_tex_next += num_bytes;} - iterator const_iterator_begin() { return m_consts.begin(); } - iterator const_iterator_end() { return m_consts.end(); } + typedef std::list<symbol*>::iterator iterator; - void dump(); + iterator global_iterator_begin() { return m_globals.begin();} + iterator global_iterator_end() { return m_globals.end();} - // Jin: handle instruction group for cdp - symbol_table *start_inst_group(); - symbol_table *end_inst_group(); + iterator const_iterator_begin() { return m_consts.begin();} + iterator const_iterator_end() { return m_consts.end();} - // backward pointer - class gpgpu_context *gpgpu_ctx; + void dump(); + + //Jin: handle instruction group for cdp + symbol_table* start_inst_group(); + symbol_table* end_inst_group(); - private: - unsigned m_reg_allocator; - unsigned m_shared_next; - unsigned m_sstarr_next; - unsigned m_const_next; - unsigned m_global_next; - unsigned m_local_next; - unsigned m_tex_next; + // backward pointer + class gpgpu_context* gpgpu_ctx; - symbol_table *m_parent; - ptx_version m_ptx_version; - std::string m_scope_name; - std::map<std::string, symbol *> - m_symbols; // map from name of register to pointers to the registers - std::map<type_info_key, type_info *, type_info_key_compare> m_types; - std::list<symbol *> m_globals; - std::list<symbol *> m_consts; - std::map<std::string, function_info *> m_function_info_lookup; - std::map<std::string, symbol_table *> m_function_symtab_lookup; +private: + unsigned m_reg_allocator; + unsigned m_shared_next; + unsigned m_sstarr_next; + unsigned m_const_next; + unsigned m_global_next; + unsigned m_local_next; + unsigned m_tex_next; - // Jin: handle instruction group for cdp - unsigned m_inst_group_id; - std::map<std::string, symbol_table *> m_inst_group_symtab; + symbol_table *m_parent; + ptx_version m_ptx_version; + std::string m_scope_name; + std::map<std::string, symbol *> m_symbols; //map from name of register to pointers to the registers + std::map<type_info_key,type_info*,type_info_key_compare> m_types; + std::list<symbol*> m_globals; + std::list<symbol*> m_consts; + std::map<std::string,function_info*> m_function_info_lookup; + std::map<std::string,symbol_table*> m_function_symtab_lookup; + + //Jin: handle instruction group for cdp + unsigned m_inst_group_id; + std::map<std::string,symbol_table*> m_inst_group_symtab; }; class operand_info { - public: - operand_info(gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = false; - m_immediate_address = false; - m_addr_offset = 0; - m_value.m_symbolic = NULL; - } - operand_info(const symbol *addr, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - if (addr->is_label()) { - m_type = label_t; - } else if (addr->is_shared()) { - m_type = symbolic_t; - } else if (addr->is_const()) { - m_type = symbolic_t; - } else if (addr->is_global()) { - m_type = symbolic_t; - } else if (addr->is_local()) { - m_type = symbolic_t; - } else if (addr->is_param_local()) { - m_type = symbolic_t; - } else if (addr->is_param_kernel()) { - m_type = symbolic_t; - } else if (addr->is_tex()) { - m_type = symbolic_t; - } else if (addr->is_func_addr()) { - m_type = symbolic_t; - } else if (!addr->is_reg()) { - m_type = symbolic_t; - } else { - m_type = reg_t; - } +public: + operand_info(gpgpu_context* ctx) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = false; + m_immediate_address=false; + m_addr_offset = 0; + m_value.m_symbolic=NULL; + } + operand_info( const symbol *addr, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + if ( addr->is_label() ) { + m_type = label_t; + } else if ( addr->is_shared() ) { + m_type = symbolic_t; + } else if ( addr->is_const() ) { + m_type = symbolic_t; + } else if ( addr->is_global() ) { + m_type = symbolic_t; + } else if ( addr->is_local() ) { + m_type = symbolic_t; + } else if ( addr->is_param_local() ) { + m_type = symbolic_t; + } else if ( addr->is_param_kernel() ) { + m_type = symbolic_t; + } else if ( addr->is_tex() ) { + m_type = symbolic_t; + } else if ( addr->is_func_addr() ) { + m_type = symbolic_t; + } else if ( !addr->is_reg() ) { + m_type = symbolic_t; + } else { + m_type = reg_t; + } + + m_is_non_arch_reg = addr->is_non_arch_reg(); + m_value.m_symbolic = addr; + m_addr_offset = 0; + m_vector = false; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + operand_info( const symbol *addr1, const symbol *addr2, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_type = memory_t; + m_value.m_vector_symbolic = new const symbol*[8]; + m_value.m_vector_symbolic[0] = addr1; + m_value.m_vector_symbolic[1] = addr2; + m_value.m_vector_symbolic[2] = NULL; + m_value.m_vector_symbolic[3] = NULL; + m_value.m_vector_symbolic[4] = NULL; + m_value.m_vector_symbolic[5] = NULL; + m_value.m_vector_symbolic[6] = NULL; + m_value.m_vector_symbolic[7] = NULL; + m_addr_offset = 0; + m_vector = false; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + operand_info( int builtin_id, int dim_mod, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = builtin_t; + m_value.m_int = builtin_id; + m_addr_offset = dim_mod; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + operand_info( const symbol *addr, int offset, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = address_t; + m_value.m_symbolic = addr; + m_addr_offset = offset; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + operand_info( unsigned x, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = unsigned_t; + m_value.m_unsigned = x; + m_addr_offset = x; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=true; + } + operand_info( int x, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = int_t; + m_value.m_int = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + operand_info( float x, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = float_op_t; + m_value.m_float = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + operand_info( double x, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = double_op_t; + m_value.m_double = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4, gpgpu_context* ctx ) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = true; + m_type = vector_t; + m_value.m_vector_symbolic = new const symbol*[8]; + m_value.m_vector_symbolic[0] = s1; + m_value.m_vector_symbolic[1] = s2; + m_value.m_vector_symbolic[2] = s3; + m_value.m_vector_symbolic[3] = s4; + m_value.m_vector_symbolic[4] = NULL; + m_value.m_vector_symbolic[5] = NULL; + m_value.m_vector_symbolic[6] = NULL; + m_value.m_vector_symbolic[7] = NULL; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4 ,const symbol *s5,const symbol *s6,const symbol *s7, const symbol *s8, gpgpu_context* ctx) + { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = true; + m_type = vector_t; + m_value.m_vector_symbolic = new const symbol*[8]; + m_value.m_vector_symbolic[0] = s1; + m_value.m_vector_symbolic[1] = s2; + m_value.m_vector_symbolic[2] = s3; + m_value.m_vector_symbolic[3] = s4; + m_value.m_vector_symbolic[4] = s5; + m_value.m_vector_symbolic[5] = s6; + m_value.m_vector_symbolic[6] = s7; + m_value.m_vector_symbolic[7] = s8; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } - m_is_non_arch_reg = addr->is_non_arch_reg(); - m_value.m_symbolic = addr; - m_addr_offset = 0; - m_vector = false; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } - operand_info(const symbol *addr1, const symbol *addr2, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_type = memory_t; - m_value.m_vector_symbolic = new const symbol *[8]; - m_value.m_vector_symbolic[0] = addr1; - m_value.m_vector_symbolic[1] = addr2; - m_value.m_vector_symbolic[2] = NULL; - m_value.m_vector_symbolic[3] = NULL; - m_value.m_vector_symbolic[4] = NULL; - m_value.m_vector_symbolic[5] = NULL; - m_value.m_vector_symbolic[6] = NULL; - m_value.m_vector_symbolic[7] = NULL; - m_addr_offset = 0; - m_vector = false; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } - operand_info(int builtin_id, int dim_mod, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = builtin_t; - m_value.m_int = builtin_id; - m_addr_offset = dim_mod; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } - operand_info(const symbol *addr, int offset, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = address_t; - m_value.m_symbolic = addr; - m_addr_offset = offset; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } - operand_info(unsigned x, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = unsigned_t; - m_value.m_unsigned = x; - m_addr_offset = x; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = true; - } - operand_info(int x, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = int_t; - m_value.m_int = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } - operand_info(float x, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = float_op_t; - m_value.m_float = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } - operand_info(double x, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = double_op_t; - m_value.m_double = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } - operand_info(const symbol *s1, const symbol *s2, const symbol *s3, - const symbol *s4, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = true; - m_type = vector_t; - m_value.m_vector_symbolic = new const symbol *[8]; - m_value.m_vector_symbolic[0] = s1; - m_value.m_vector_symbolic[1] = s2; - m_value.m_vector_symbolic[2] = s3; - m_value.m_vector_symbolic[3] = s4; - m_value.m_vector_symbolic[4] = NULL; - m_value.m_vector_symbolic[5] = NULL; - m_value.m_vector_symbolic[6] = NULL; - m_value.m_vector_symbolic[7] = NULL; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } - operand_info(const symbol *s1, const symbol *s2, const symbol *s3, - const symbol *s4, const symbol *s5, const symbol *s6, - const symbol *s7, const symbol *s8, gpgpu_context *ctx) { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = true; - m_type = vector_t; - m_value.m_vector_symbolic = new const symbol *[8]; - m_value.m_vector_symbolic[0] = s1; - m_value.m_vector_symbolic[1] = s2; - m_value.m_vector_symbolic[2] = s3; - m_value.m_vector_symbolic[3] = s4; - m_value.m_vector_symbolic[4] = s5; - m_value.m_vector_symbolic[5] = s6; - m_value.m_vector_symbolic[6] = s7; - m_value.m_vector_symbolic[7] = s8; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address = false; - } + void init(gpgpu_context* ctx) + { + gpgpu_ctx = ctx; + m_uid=(unsigned)-1; + m_valid=false; + m_vector=false; + m_type=undef_t; + m_immediate_address=false; + m_addr_space=undefined_space; + m_operand_lohi=0; + m_double_operand_type=0; + m_operand_neg=false; + m_const_mem_offset=(unsigned)-1; + m_value.m_int=0; + m_value.m_unsigned=(unsigned)-1; + m_value.m_float=0; + m_value.m_double=0; + for(unsigned i=0; i<4; i++){ + m_value.m_vint[i]=0; + m_value.m_vunsigned[i]=0; + m_value.m_vfloat[i]=0; + m_value.m_vdouble[i]=0; + } + m_value.m_symbolic=NULL; + m_value.m_vector_symbolic=NULL; + m_addr_offset=0; + m_neg_pred=0; + m_is_return_var=0; + m_is_non_arch_reg=0; - void init(gpgpu_context *ctx) { - gpgpu_ctx = ctx; - m_uid = (unsigned)-1; - m_valid = false; - m_vector = false; - m_type = undef_t; - m_immediate_address = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = (unsigned)-1; - m_value.m_int = 0; - m_value.m_unsigned = (unsigned)-1; - m_value.m_float = 0; - m_value.m_double = 0; - for (unsigned i = 0; i < 4; i++) { - m_value.m_vint[i] = 0; - m_value.m_vunsigned[i] = 0; - m_value.m_vfloat[i] = 0; - m_value.m_vdouble[i] = 0; - } - m_value.m_symbolic = NULL; - m_value.m_vector_symbolic = NULL; - m_addr_offset = 0; - m_neg_pred = 0; - m_is_return_var = 0; - m_is_non_arch_reg = 0; - } - void make_memory_operand() { m_type = memory_t; } - void set_return() { m_is_return_var = true; } - void set_immediate_addr() { m_immediate_address = true; } - const std::string &name() const { - assert(m_type == symbolic_t || m_type == reg_t || m_type == address_t || - m_type == memory_t || m_type == label_t); - return m_value.m_symbolic->name(); - } + } + void make_memory_operand() { m_type = memory_t;} + void set_return() { m_is_return_var = true; } + void set_immediate_addr() {m_immediate_address=true;} + const std::string &name() const + { + assert( m_type == symbolic_t || m_type == reg_t || m_type == address_t || m_type == memory_t || m_type == label_t); + return m_value.m_symbolic->name(); + } - unsigned get_vect_nelem() const { - assert(is_vector()); - if (!m_value.m_vector_symbolic[0]) return 0; - if (!m_value.m_vector_symbolic[1]) return 1; - if (!m_value.m_vector_symbolic[2]) return 2; - if (!m_value.m_vector_symbolic[3]) return 3; - if (!m_value.m_vector_symbolic[4]) return 4; - if (!m_value.m_vector_symbolic[5]) return 5; - if (!m_value.m_vector_symbolic[6]) return 6; - if (!m_value.m_vector_symbolic[7]) return 7; - return 8; - } + unsigned get_vect_nelem() const + { + assert( is_vector() ); + if( !m_value.m_vector_symbolic[0] ) return 0; + if( !m_value.m_vector_symbolic[1] ) return 1; + if( !m_value.m_vector_symbolic[2] ) return 2; + if( !m_value.m_vector_symbolic[3] ) return 3; + if( !m_value.m_vector_symbolic[4] ) return 4; + if( !m_value.m_vector_symbolic[5] ) return 5; + if( !m_value.m_vector_symbolic[6] ) return 6; + if( !m_value.m_vector_symbolic[7] ) return 7; + return 8; + } - const symbol *vec_symbol(int idx) const { - assert(idx < 8); - const symbol *result = m_value.m_vector_symbolic[idx]; - assert(result != NULL); - return result; - } + const symbol* vec_symbol(int idx) const + { + assert(idx < 8); + const symbol *result = m_value.m_vector_symbolic[idx]; + assert( result != NULL ); + return result; + } - const std::string &vec_name1() const { - assert(m_type == vector_t); - return m_value.m_vector_symbolic[0]->name(); - } + const std::string &vec_name1() const + { + assert( m_type == vector_t); + return m_value.m_vector_symbolic[0]->name(); + } - const std::string &vec_name2() const { - assert(m_type == vector_t); - return m_value.m_vector_symbolic[1]->name(); - } + const std::string &vec_name2() const + { + assert( m_type == vector_t); + return m_value.m_vector_symbolic[1]->name(); + } - const std::string &vec_name3() const { - assert(m_type == vector_t); - return m_value.m_vector_symbolic[2]->name(); - } + const std::string &vec_name3() const + { + assert( m_type == vector_t); + return m_value.m_vector_symbolic[2]->name(); + } - const std::string &vec_name4() const { - assert(m_type == vector_t); - return m_value.m_vector_symbolic[3]->name(); - } + const std::string &vec_name4() const + { + assert( m_type == vector_t); + return m_value.m_vector_symbolic[3]->name(); + } - bool is_reg() const { - if (m_type == reg_t) { - return true; - } - if (m_type != symbolic_t) { - return false; - } - return m_value.m_symbolic->type()->get_key().is_reg(); - } - bool is_param_local() const { - if (m_type != symbolic_t) return false; - return m_value.m_symbolic->type()->get_key().is_param_local(); - } + bool is_reg() const + { + if ( m_type == reg_t ) { + return true; + } + if ( m_type != symbolic_t ) { + return false; + } + return m_value.m_symbolic->type()->get_key().is_reg(); + } + bool is_param_local() const + { + if ( m_type != symbolic_t ) + return false; + return m_value.m_symbolic->type()->get_key().is_param_local(); + } - bool is_param_kernel() const { - if (m_type != symbolic_t) return false; - return m_value.m_symbolic->type()->get_key().is_param_kernel(); - } + bool is_param_kernel() const + { + if ( m_type != symbolic_t ) + return false; + return m_value.m_symbolic->type()->get_key().is_param_kernel(); + } - bool is_vector() const { - if (m_vector) return true; - return false; - } - int reg_num() const { return m_value.m_symbolic->reg_num(); } - int reg1_num() const { return m_value.m_vector_symbolic[0]->reg_num(); } - int reg2_num() const { return m_value.m_vector_symbolic[1]->reg_num(); } - int reg3_num() const { - return m_value.m_vector_symbolic[2] - ? m_value.m_vector_symbolic[2]->reg_num() - : 0; - } - int reg4_num() const { - return m_value.m_vector_symbolic[3] - ? m_value.m_vector_symbolic[3]->reg_num() - : 0; - } - int reg5_num() const { - return m_value.m_vector_symbolic[4] - ? m_value.m_vector_symbolic[4]->reg_num() - : 0; - } - int reg6_num() const { - return m_value.m_vector_symbolic[5] - ? m_value.m_vector_symbolic[5]->reg_num() - : 0; - } - int reg7_num() const { - return m_value.m_vector_symbolic[6] - ? m_value.m_vector_symbolic[6]->reg_num() - : 0; - } - int reg8_num() const { - return m_value.m_vector_symbolic[7] - ? m_value.m_vector_symbolic[7]->reg_num() - : 0; - } - int arch_reg_num() const { return m_value.m_symbolic->arch_reg_num(); } - int arch_reg_num(unsigned n) const { - return (m_value.m_vector_symbolic[n]) - ? m_value.m_vector_symbolic[n]->arch_reg_num() - : -1; - } - bool is_label() const { return m_type == label_t; } - bool is_builtin() const { return m_type == builtin_t; } + bool is_vector() const + { + if ( m_vector) return true; + return false; + } + int reg_num() const { return m_value.m_symbolic->reg_num();} + int reg1_num() const { return m_value.m_vector_symbolic[0]->reg_num();} + int reg2_num() const { return m_value.m_vector_symbolic[1]->reg_num();} + int reg3_num() const { return m_value.m_vector_symbolic[2]?m_value.m_vector_symbolic[2]->reg_num():0; } + int reg4_num() const { return m_value.m_vector_symbolic[3]?m_value.m_vector_symbolic[3]->reg_num():0; } + int reg5_num() const { return m_value.m_vector_symbolic[4]?m_value.m_vector_symbolic[4]->reg_num():0; } + int reg6_num() const { return m_value.m_vector_symbolic[5]?m_value.m_vector_symbolic[5]->reg_num():0; } + int reg7_num() const { return m_value.m_vector_symbolic[6]?m_value.m_vector_symbolic[6]->reg_num():0; } + int reg8_num() const { return m_value.m_vector_symbolic[7]?m_value.m_vector_symbolic[7]->reg_num():0; } + int arch_reg_num() const { return m_value.m_symbolic->arch_reg_num(); } + int arch_reg_num(unsigned n) const { return (m_value.m_vector_symbolic[n])? m_value.m_vector_symbolic[n]->arch_reg_num() : -1; } + bool is_label() const { return m_type == label_t;} + bool is_builtin() const { return m_type == builtin_t;} - // Memory operand used in ld / st instructions (ex. [__var1]) - bool is_memory_operand() const { return m_type == memory_t; } + // Memory operand used in ld / st instructions (ex. [__var1]) + bool is_memory_operand() const { return m_type == memory_t;} - // Memory operand with immediate access (ex. s[0x0004] or g[$r1+=0x0004]) - // This is used by the PTXPlus extension. The operand is assigned an address - // space during parsing. - bool is_memory_operand2() const { return (m_addr_space != undefined_space); } + // Memory operand with immediate access (ex. s[0x0004] or g[$r1+=0x0004]) + // This is used by the PTXPlus extension. The operand is assigned an address space during parsing. + bool is_memory_operand2() const { + return (m_addr_space!=undefined_space); + } - bool is_immediate_address() const { return m_immediate_address; } + bool is_immediate_address() const { + return m_immediate_address; + } - bool is_literal() const { - return m_type == int_t || m_type == float_op_t || m_type == double_op_t || - m_type == unsigned_t; - } - bool is_shared() const { - if (!(m_type == symbolic_t || m_type == address_t || m_type == memory_t)) { - return false; - } - return m_value.m_symbolic->is_shared(); - } - bool is_sstarr() const { return m_value.m_symbolic->is_sstarr(); } - bool is_const() const { return m_value.m_symbolic->is_const(); } - bool is_global() const { return m_value.m_symbolic->is_global(); } - bool is_local() const { return m_value.m_symbolic->is_local(); } - bool is_tex() const { return m_value.m_symbolic->is_tex(); } - bool is_return_var() const { return m_is_return_var; } + bool is_literal() const { return m_type == int_t || + m_type == float_op_t || + m_type == double_op_t || + m_type == unsigned_t;} + bool is_shared() const { + if ( !(m_type == symbolic_t || m_type == address_t || m_type == memory_t) ) { + return false; + } + return m_value.m_symbolic->is_shared(); + } + bool is_sstarr() const { return m_value.m_symbolic->is_sstarr();} + bool is_const() const { return m_value.m_symbolic->is_const();} + bool is_global() const { return m_value.m_symbolic->is_global();} + bool is_local() const { return m_value.m_symbolic->is_local();} + bool is_tex() const { return m_value.m_symbolic->is_tex();} + bool is_return_var() const { return m_is_return_var; } - bool is_function_address() const { - if (m_type != symbolic_t) { - return false; - } - return m_value.m_symbolic->is_func_addr(); - } + bool is_function_address() const + { + if( m_type != symbolic_t ) { + return false; + } + return m_value.m_symbolic->is_func_addr(); + } - ptx_reg_t get_literal_value() const { - ptx_reg_t result; - switch (m_type) { - case int_t: - result.s64 = m_value.m_int; - break; - case float_op_t: - result.f32 = m_value.m_float; - break; - case double_op_t: - result.f64 = m_value.m_double; - break; - case unsigned_t: - result.u32 = m_value.m_unsigned; - break; + ptx_reg_t get_literal_value() const + { + ptx_reg_t result; + switch ( m_type ) { + case int_t: result.s64 = m_value.m_int; break; + case float_op_t: result.f32 = m_value.m_float; break; + case double_op_t: result.f64 = m_value.m_double; break; + case unsigned_t: result.u32 = m_value.m_unsigned; break; default: - assert(0); - break; - } - return result; - } - int get_int() const { return m_value.m_int; } - int get_addr_offset() const { return m_addr_offset; } - const symbol *get_symbol() const { return m_value.m_symbolic; } - void set_type(enum operand_type type) { m_type = type; } - enum operand_type get_type() const { return m_type; } - void set_neg_pred() { - assert(m_valid); - m_neg_pred = true; - } - bool is_neg_pred() const { return m_neg_pred; } - bool is_valid() const { return m_valid; } + assert(0); + break; + } + return result; + } + int get_int() const { return m_value.m_int;} + int get_addr_offset() const { return m_addr_offset;} + const symbol *get_symbol() const { return m_value.m_symbolic;} + void set_type( enum operand_type type ) + { + m_type = type; + } + enum operand_type get_type() const { + return m_type; + } + void set_neg_pred() + { + assert( m_valid ); + m_neg_pred = true; + } + bool is_neg_pred() const { return m_neg_pred; } + bool is_valid() const { return m_valid; } - void set_addr_space(enum _memory_space_t set_value) { - m_addr_space = set_value; - } - enum _memory_space_t get_addr_space() const { return m_addr_space; } - void set_operand_lohi(int set_value) { m_operand_lohi = set_value; } - int get_operand_lohi() const { return m_operand_lohi; } - void set_double_operand_type(int set_value) { - m_double_operand_type = set_value; - } - int get_double_operand_type() const { return m_double_operand_type; } - void set_operand_neg() { m_operand_neg = true; } - bool get_operand_neg() const { return m_operand_neg; } - void set_const_mem_offset(addr_t set_value) { - m_const_mem_offset = set_value; - } - addr_t get_const_mem_offset() const { return m_const_mem_offset; } - bool is_non_arch_reg() const { return m_is_non_arch_reg; } + void set_addr_space(enum _memory_space_t set_value) { m_addr_space = set_value; } + enum _memory_space_t get_addr_space() const { return m_addr_space; } + void set_operand_lohi(int set_value) { m_operand_lohi = set_value; } + int get_operand_lohi() const { return m_operand_lohi; } + void set_double_operand_type(int set_value) { m_double_operand_type = set_value; } + int get_double_operand_type() const { return m_double_operand_type; } + void set_operand_neg() { m_operand_neg = true; } + bool get_operand_neg() const { return m_operand_neg; } + void set_const_mem_offset(addr_t set_value) { m_const_mem_offset = set_value; } + addr_t get_const_mem_offset() const { return m_const_mem_offset; } + bool is_non_arch_reg() const { return m_is_non_arch_reg; } - private: - gpgpu_context *gpgpu_ctx; - unsigned m_uid; - bool m_valid; - bool m_vector; - enum operand_type m_type; - bool m_immediate_address; - enum _memory_space_t m_addr_space; - int m_operand_lohi; - int m_double_operand_type; - bool m_operand_neg; - addr_t m_const_mem_offset; - union { - int m_int; - unsigned int m_unsigned; - float m_float; - double m_double; - int m_vint[4]; - unsigned int m_vunsigned[4]; - float m_vfloat[4]; - double m_vdouble[4]; - const symbol *m_symbolic; - const symbol **m_vector_symbolic; - } m_value; +private: + gpgpu_context* gpgpu_ctx; + unsigned m_uid; + bool m_valid; + bool m_vector; + enum operand_type m_type; + bool m_immediate_address; + enum _memory_space_t m_addr_space; + int m_operand_lohi; + int m_double_operand_type; + bool m_operand_neg; + addr_t m_const_mem_offset; + union { + int m_int; + unsigned int m_unsigned; + float m_float; + double m_double; + int m_vint[4]; + unsigned int m_vunsigned[4]; + float m_vfloat[4]; + double m_vdouble[4]; + const symbol* m_symbolic; + const symbol** m_vector_symbolic; + } m_value; - int m_addr_offset; + int m_addr_offset; - bool m_neg_pred; - bool m_is_return_var; - bool m_is_non_arch_reg; + bool m_neg_pred; + bool m_is_return_var; + bool m_is_non_arch_reg; - unsigned get_uid(); + unsigned get_uid(); }; extern const char *g_opcode_string[]; struct basic_block_t { - basic_block_t(unsigned ID, ptx_instruction *begin, ptx_instruction *end, - bool entry, bool ex) { - bb_id = ID; - ptx_begin = begin; - ptx_end = end; - is_entry = entry; - is_exit = ex; - immediatepostdominator_id = -1; - immediatedominator_id = -1; - } + basic_block_t( unsigned ID, ptx_instruction *begin, ptx_instruction *end, bool entry, bool ex) + { + bb_id = ID; + ptx_begin = begin; + ptx_end = end; + is_entry=entry; + is_exit=ex; + immediatepostdominator_id = -1; + immediatedominator_id = -1; + } - ptx_instruction *ptx_begin; - ptx_instruction *ptx_end; - std::set<int> - predecessor_ids; // indices of other basic blocks in m_basic_blocks array - std::set<int> successor_ids; - std::set<int> postdominator_ids; - std::set<int> dominator_ids; - std::set<int> Tmp_ids; - int immediatepostdominator_id; - int immediatedominator_id; - bool is_entry; - bool is_exit; - unsigned bb_id; + ptx_instruction* ptx_begin; + ptx_instruction* ptx_end; + std::set<int> predecessor_ids; //indices of other basic blocks in m_basic_blocks array + std::set<int> successor_ids; + std::set<int> postdominator_ids; + std::set<int> dominator_ids; + std::set<int> Tmp_ids; + int immediatepostdominator_id; + int immediatedominator_id; + bool is_entry; + bool is_exit; + unsigned bb_id; - // if this basic block dom B - bool dom(const basic_block_t *B) { - return (B->dominator_ids.find(this->bb_id) != B->dominator_ids.end()); - } + // if this basic block dom B + bool dom(const basic_block_t *B) { + return (B->dominator_ids.find(this->bb_id) != B->dominator_ids.end()); + } - // if this basic block pdom B - bool pdom(const basic_block_t *B) { - return (B->postdominator_ids.find(this->bb_id) != - B->postdominator_ids.end()); - } + // if this basic block pdom B + bool pdom(const basic_block_t *B) { + return (B->postdominator_ids.find(this->bb_id) != B->postdominator_ids.end()); + } }; struct gpgpu_recon_t { - address_type source_pc; - address_type target_pc; - class ptx_instruction *source_inst; - class ptx_instruction *target_inst; + address_type source_pc; + address_type target_pc; + class ptx_instruction* source_inst; + class ptx_instruction* target_inst; }; class ptx_instruction : public warp_inst_t { - public: - ptx_instruction(int opcode, const symbol *pred, int neg_pred, int pred_mod, - symbol *label, const std::list<operand_info> &operands, - const operand_info &return_var, const std::list<int> &options, - const std::list<int> &wmma_options, - const std::list<int> &scalar_type, memory_space_t space_spec, - const char *file, unsigned line, const char *source, - const core_config *config, gpgpu_context *ctx); +public: + ptx_instruction( int opcode, + const symbol *pred, + int neg_pred, + int pred_mod, + symbol *label, + const std::list<operand_info> &operands, + const operand_info &return_var, + const std::list<int> &options, + const std::list<int> &wmma_options, + const std::list<int> &scalar_type, + memory_space_t space_spec, + const char *file, + unsigned line, + const char *source, + const core_config *config, + gpgpu_context* ctx); - void print_insn() const; - virtual void print_insn(FILE *fp) const; - std::string to_string() const; - unsigned inst_size() const { return m_inst_size; } - unsigned uid() const { return m_uid; } - int get_opcode() const { return m_opcode; } - const char *get_opcode_cstr() const { - if (m_opcode != -1) { - return g_opcode_string[m_opcode]; - } else { - return "label"; - } - } - const char *source_file() const { return m_source_file.c_str(); } - unsigned source_line() const { return m_source_line; } - unsigned get_num_operands() const { return m_operands.size(); } - bool has_pred() const { return m_pred != NULL; } - operand_info get_pred() const; - bool get_pred_neg() const { return m_neg_pred; } - int get_pred_mod() const { return m_pred_mod; } - const char *get_source() const { return m_source.c_str(); } + void print_insn() const; + virtual void print_insn( FILE *fp ) const; + std::string to_string() const; + unsigned inst_size() const { return m_inst_size; } + unsigned uid() const { return m_uid;} + int get_opcode() const { return m_opcode;} + const char *get_opcode_cstr() const + { + if ( m_opcode != -1 ) { + return g_opcode_string[m_opcode]; + } else { + return "label"; + } + } + const char *source_file() const { return m_source_file.c_str();} + unsigned source_line() const { return m_source_line;} + unsigned get_num_operands() const { return m_operands.size();} + bool has_pred() const { return m_pred != NULL;} + operand_info get_pred() const; + bool get_pred_neg() const { return m_neg_pred;} + int get_pred_mod() const { return m_pred_mod;} + const char *get_source() const { return m_source.c_str();} - typedef std::vector<operand_info>::const_iterator const_iterator; + typedef std::vector<operand_info>::const_iterator const_iterator; - const_iterator op_iter_begin() const { return m_operands.begin(); } + const_iterator op_iter_begin() const + { + return m_operands.begin(); + } - const_iterator op_iter_end() const { return m_operands.end(); } + const_iterator op_iter_end() const + { + return m_operands.end(); + } - const operand_info &dst() const { - assert(!m_operands.empty()); - return m_operands[0]; - } + const operand_info &dst() const + { + assert( !m_operands.empty() ); + return m_operands[0]; + } - const operand_info &func_addr() const { - assert(!m_operands.empty()); - if (!m_operands[0].is_return_var()) { + const operand_info &func_addr() const + { + assert( !m_operands.empty() ); + if( !m_operands[0].is_return_var() ) { + return m_operands[0]; + } else { + assert( m_operands.size() >= 2 ); + return m_operands[1]; + } + } + + operand_info &dst() + { + assert( !m_operands.empty() ); return m_operands[0]; - } else { - assert(m_operands.size() >= 2); - return m_operands[1]; - } - } + } - operand_info &dst() { - assert(!m_operands.empty()); - return m_operands[0]; - } + const operand_info &src1() const + { + assert( m_operands.size() > 1 ); + return m_operands[1]; + } - const operand_info &src1() const { - assert(m_operands.size() > 1); - return m_operands[1]; - } + const operand_info &src2() const + { + assert( m_operands.size() > 2 ); + return m_operands[2]; + } - const operand_info &src2() const { - assert(m_operands.size() > 2); - return m_operands[2]; - } + const operand_info &src3() const + { + assert( m_operands.size() > 3 ); + return m_operands[3]; + } + const operand_info &src4() const + { + assert( m_operands.size() > 4 ); + return m_operands[4]; + } + const operand_info &src5() const + { + assert( m_operands.size() > 5 ); + return m_operands[5]; + } + const operand_info &src6() const + { + assert( m_operands.size() > 6 ); + return m_operands[6]; + } + const operand_info &src7() const + { + assert( m_operands.size() > 7 ); + return m_operands[7]; + } + const operand_info &src8() const + { + assert( m_operands.size() > 8 ); + return m_operands[8]; + } - const operand_info &src3() const { - assert(m_operands.size() > 3); - return m_operands[3]; - } - const operand_info &src4() const { - assert(m_operands.size() > 4); - return m_operands[4]; - } - const operand_info &src5() const { - assert(m_operands.size() > 5); - return m_operands[5]; - } - const operand_info &src6() const { - assert(m_operands.size() > 6); - return m_operands[6]; - } - const operand_info &src7() const { - assert(m_operands.size() > 7); - return m_operands[7]; - } - const operand_info &src8() const { - assert(m_operands.size() > 8); - return m_operands[8]; - } + const operand_info &operand_lookup( unsigned n ) const + { + assert( n < m_operands.size() ); + return m_operands[n]; + } + bool has_return() const + { + return m_return_var.is_valid(); + } - const operand_info &operand_lookup(unsigned n) const { - assert(n < m_operands.size()); - return m_operands[n]; - } - bool has_return() const { return m_return_var.is_valid(); } + memory_space_t get_space() const { return m_space_spec;} + unsigned get_vector() const { return m_vector_spec;} + unsigned get_atomic() const { return m_atomic_spec;} - memory_space_t get_space() const { return m_space_spec; } - unsigned get_vector() const { return m_vector_spec; } - unsigned get_atomic() const { return m_atomic_spec; } + int get_wmma_type() const { + return m_wmma_type; + } + int get_wmma_layout(int index) const { + return m_wmma_layout[index];//0->Matrix D,1->Matrix C + } + int get_type() const + { + assert( !m_scalar_type.empty() ); + return m_scalar_type.front(); + } - int get_wmma_type() const { return m_wmma_type; } - int get_wmma_layout(int index) const { - return m_wmma_layout[index]; // 0->Matrix D,1->Matrix C - } - int get_type() const { - assert(!m_scalar_type.empty()); - return m_scalar_type.front(); - } + int get_type2() const + { + assert( m_scalar_type.size()==2 ); + return m_scalar_type.back(); + } - int get_type2() const { - assert(m_scalar_type.size() == 2); - return m_scalar_type.back(); - } + void assign_bb(basic_block_t* basic_block) //assign instruction to a basic block + { + m_basic_block = basic_block; + } + basic_block_t* get_bb() { return m_basic_block;} + void set_m_instr_mem_index(unsigned index) { + m_instr_mem_index = index; + } + void set_PC( addr_t PC ) + { + m_PC = PC; + } + addr_t get_PC() const + { + return m_PC; + } - void assign_bb( - basic_block_t *basic_block) // assign instruction to a basic block - { - m_basic_block = basic_block; - } - basic_block_t *get_bb() { return m_basic_block; } - void set_m_instr_mem_index(unsigned index) { m_instr_mem_index = index; } - void set_PC(addr_t PC) { m_PC = PC; } - addr_t get_PC() const { return m_PC; } + unsigned get_m_instr_mem_index() { return m_instr_mem_index;} + unsigned get_cmpop() const { return m_compare_op;} + const symbol *get_label() const { return m_label;} + bool is_label() const { if(m_label){ assert(m_opcode==-1);return true;} return false;} + bool is_hi() const { return m_hi;} + bool is_lo() const { return m_lo;} + bool is_wide() const { return m_wide;} + bool is_uni() const { return m_uni;} + bool is_exit() const { return m_exit;} + bool is_abs() const { return m_abs;} + bool is_neg() const { return m_neg;} + bool is_to() const { return m_to_option; } + unsigned cache_option() const { return m_cache_option; } + unsigned rounding_mode() const { return m_rounding_mode;} + unsigned saturation_mode() const { return m_saturation_mode;} + unsigned dimension() const { return m_geom_spec;} + unsigned barrier_op() const {return m_barrier_op;} + unsigned shfl_op() const {return m_shfl_op;} + unsigned prmt_op() const {return m_prmt_op;} + enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot }; + enum vote_mode_t vote_mode() const { return m_vote_mode; } - unsigned get_m_instr_mem_index() { return m_instr_mem_index; } - unsigned get_cmpop() const { return m_compare_op; } - const symbol *get_label() const { return m_label; } - bool is_label() const { - if (m_label) { - assert(m_opcode == -1); - return true; - } - return false; - } - bool is_hi() const { return m_hi; } - bool is_lo() const { return m_lo; } - bool is_wide() const { return m_wide; } - bool is_uni() const { return m_uni; } - bool is_exit() const { return m_exit; } - bool is_abs() const { return m_abs; } - bool is_neg() const { return m_neg; } - bool is_to() const { return m_to_option; } - unsigned cache_option() const { return m_cache_option; } - unsigned rounding_mode() const { return m_rounding_mode; } - unsigned saturation_mode() const { return m_saturation_mode; } - unsigned dimension() const { return m_geom_spec; } - unsigned barrier_op() const { return m_barrier_op; } - unsigned shfl_op() const { return m_shfl_op; } - unsigned prmt_op() const { return m_prmt_op; } - enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot }; - enum vote_mode_t vote_mode() const { return m_vote_mode; } + int membar_level() const { return m_membar_level; } - int membar_level() const { return m_membar_level; } + bool has_memory_read() const { + if( m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP|| m_opcode==MMA_LD_OP ) + return true; + // Check PTXPlus operand type below + // Source operands are memory operands + ptx_instruction::const_iterator op=op_iter_begin(); + for ( int n=0; op != op_iter_end(); op++, n++ ) { //process operands + if( n > 0 && op->is_memory_operand2()) // source operands only + return true; + } + return false; + } + bool has_memory_write() const { + if( m_opcode == ST_OP || m_opcode==MMA_ST_OP ) return true; + // Check PTXPlus operand type below + // Destination operand is a memory operand + ptx_instruction::const_iterator op=op_iter_begin(); + for ( int n=0; (op!=op_iter_end() && n<1); op++, n++ ) { //process operands + if( n==0 && op->is_memory_operand2()) // source operands only + return true; + } + return false; + } - bool has_memory_read() const { - if (m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP || - m_opcode == MMA_LD_OP) - return true; - // Check PTXPlus operand type below - // Source operands are memory operands - ptx_instruction::const_iterator op = op_iter_begin(); - for (int n = 0; op != op_iter_end(); op++, n++) { // process operands - if (n > 0 && op->is_memory_operand2()) // source operands only - return true; - } - return false; - } - bool has_memory_write() const { - if (m_opcode == ST_OP || m_opcode == MMA_ST_OP) return true; - // Check PTXPlus operand type below - // Destination operand is a memory operand - ptx_instruction::const_iterator op = op_iter_begin(); - for (int n = 0; (op != op_iter_end() && n < 1); - op++, n++) { // process operands - if (n == 0 && op->is_memory_operand2()) // source operands only - return true; - } - return false; - } - private: - void set_opcode_and_latency(); - void set_bar_type(); - void set_fp_or_int_archop(); - void set_mul_div_or_other_archop(); +private: + void set_opcode_and_latency(); + void set_bar_type(); + void set_fp_or_int_archop(); + void set_mul_div_or_other_archop(); - basic_block_t *m_basic_block; - unsigned m_uid; - addr_t m_PC; - std::string m_source_file; - unsigned m_source_line; - std::string m_source; + basic_block_t *m_basic_block; + unsigned m_uid; + addr_t m_PC; + std::string m_source_file; + unsigned m_source_line; + std::string m_source; - const symbol *m_pred; - bool m_neg_pred; - int m_pred_mod; - int m_opcode; - const symbol *m_label; - std::vector<operand_info> m_operands; - operand_info m_return_var; + const symbol *m_pred; + bool m_neg_pred; + int m_pred_mod; + int m_opcode; + const symbol *m_label; + std::vector<operand_info> m_operands; + operand_info m_return_var; - std::list<int> m_options; - std::list<int> m_wmma_options; - bool m_wide; - bool m_hi; - bool m_lo; - bool m_exit; - bool m_abs; - bool m_neg; - bool m_uni; // if branch instruction, this evaluates to true for uniform - // branches (ie jumps) - bool m_to_option; - unsigned m_cache_option; - int m_wmma_type; - int m_wmma_layout[2]; - int m_wmma_configuration; - unsigned m_rounding_mode; - unsigned m_compare_op; - unsigned m_saturation_mode; - unsigned m_barrier_op; - unsigned m_shfl_op; - unsigned m_prmt_op; + std::list<int> m_options; + std::list<int> m_wmma_options; + bool m_wide; + bool m_hi; + bool m_lo; + bool m_exit; + bool m_abs; + bool m_neg; + bool m_uni; //if branch instruction, this evaluates to true for uniform branches (ie jumps) + bool m_to_option; + unsigned m_cache_option; + int m_wmma_type; + int m_wmma_layout[2]; + int m_wmma_configuration; + unsigned m_rounding_mode; + unsigned m_compare_op; + unsigned m_saturation_mode; + unsigned m_barrier_op; + unsigned m_shfl_op; + unsigned m_prmt_op; - std::list<int> m_scalar_type; - memory_space_t m_space_spec; - int m_geom_spec; - int m_vector_spec; - int m_atomic_spec; - enum vote_mode_t m_vote_mode; - int m_membar_level; - int m_instr_mem_index; // index into m_instr_mem array - unsigned m_inst_size; // bytes + std::list<int> m_scalar_type; + memory_space_t m_space_spec; + int m_geom_spec; + int m_vector_spec; + int m_atomic_spec; + enum vote_mode_t m_vote_mode; + int m_membar_level; + int m_instr_mem_index; //index into m_instr_mem array + unsigned m_inst_size; // bytes - virtual void pre_decode(); - friend class function_info; - // backward pointer - class gpgpu_context *gpgpu_ctx; + virtual void pre_decode(); + friend class function_info; + // backward pointer + class gpgpu_context* gpgpu_ctx; }; class param_info { - public: - param_info() { - m_valid = false; - m_value_set = false; - m_size = 0; - m_is_ptr = false; - } - param_info(std::string name, int type, size_t size, bool is_ptr, - memory_space_t ptr_space) { - m_valid = true; - m_value_set = false; - m_name = name; - m_type = type; - m_size = size; - m_is_ptr = is_ptr; - m_ptr_space = ptr_space; - } - void add_data(param_t v) { - assert((!m_value_set) || (m_value.size == v.size)); // if this fails - // concurrent kernel - // launches might - // execute incorrectly - m_value_set = true; - m_value = v; - } - void add_offset(unsigned offset) { m_offset = offset; } - unsigned get_offset() { - assert(m_valid); - return m_offset; - } - std::string get_name() const { - assert(m_valid); - return m_name; - } - int get_type() const { - assert(m_valid); - return m_type; - } - param_t get_value() const { - assert(m_value_set); - return m_value; - } - size_t get_size() const { - assert(m_valid); - return m_size; - } - bool is_ptr_shared() const { - assert(m_valid); - return (m_is_ptr and m_ptr_space == shared_space); - } - - private: - bool m_valid; - std::string m_name; - int m_type; - size_t m_size; - bool m_value_set; - param_t m_value; - unsigned m_offset; - bool m_is_ptr; - memory_space_t m_ptr_space; +public: + param_info() { m_valid = false; m_value_set=false; m_size = 0; m_is_ptr = false; } + param_info( std::string name, int type, size_t size, bool is_ptr, memory_space_t ptr_space ) + { + m_valid = true; + m_value_set = false; + m_name = name; + m_type = type; + m_size = size; + m_is_ptr = is_ptr; + m_ptr_space = ptr_space; + } + void add_data( param_t v ) { + assert( (!m_value_set) || (m_value.size == v.size) ); // if this fails concurrent kernel launches might execute incorrectly + m_value_set = true; + m_value = v; + } + void add_offset( unsigned offset ) { m_offset = offset; } + unsigned get_offset() { assert(m_valid); return m_offset; } + std::string get_name() const { assert(m_valid); return m_name; } + int get_type() const { assert(m_valid); return m_type; } + param_t get_value() const { assert(m_value_set); return m_value; } + size_t get_size() const { assert(m_valid); return m_size; } + bool is_ptr_shared() const { assert(m_valid); return (m_is_ptr and m_ptr_space == shared_space); } +private: + bool m_valid; + std::string m_name; + int m_type; + size_t m_size; + bool m_value_set; + param_t m_value; + unsigned m_offset; + bool m_is_ptr; + memory_space_t m_ptr_space; }; class function_info { - public: - function_info(int entry_point, gpgpu_context *ctx); - const ptx_version &get_ptx_version() const { - return m_symtab->get_ptx_version(); - } - unsigned get_sm_target() const { return m_symtab->get_sm_target(); } - bool is_extern() const { return m_extern; } - void set_name(const char *name) { m_name = name; } - void set_symtab(symbol_table *symtab) { m_symtab = symtab; } - std::string get_name() const { return m_name; } - unsigned print_insn(unsigned pc, FILE *fp) const; - std::string get_insn_str(unsigned pc) const; - void add_inst(const std::list<ptx_instruction *> &instructions) { - m_instructions = instructions; - } - std::list<ptx_instruction *>::iterator find_next_real_instruction( - std::list<ptx_instruction *>::iterator i); - void create_basic_blocks(); - - void print_basic_blocks(); +public: + function_info(int entry_point, gpgpu_context* ctx ); + const ptx_version &get_ptx_version() const { return m_symtab->get_ptx_version(); } + unsigned get_sm_target() const { return m_symtab->get_sm_target(); } + bool is_extern() const { return m_extern; } + void set_name(const char *name) + { + m_name = name; + } + void set_symtab(symbol_table *symtab ) + { + m_symtab = symtab; + } + std::string get_name() const + { + return m_name; + } + unsigned print_insn( unsigned pc, FILE * fp ) const; + std::string get_insn_str( unsigned pc ) const; + void add_inst( const std::list<ptx_instruction*> &instructions ) + { + m_instructions = instructions; + } + std::list<ptx_instruction*>::iterator find_next_real_instruction( std::list<ptx_instruction*>::iterator i ); + void create_basic_blocks( ); - void print_basic_block_links(); - void print_basic_block_dot(); + void print_basic_blocks(); - operand_info *find_break_target( - ptx_instruction *p_break_insn); // find the target of a break instruction - void connect_basic_blocks(); // iterate across m_basic_blocks of function, - // connecting basic blocks together - bool - connect_break_targets(); // connecting break instructions with proper targets + void print_basic_block_links(); + void print_basic_block_dot(); - // iterate across m_basic_blocks of function, - // finding dominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - void find_dominators(); - void print_dominators(); - void find_idominators(); - void print_idominators(); + operand_info* find_break_target( ptx_instruction * p_break_insn ); //find the target of a break instruction + void connect_basic_blocks( ); //iterate across m_basic_blocks of function, connecting basic blocks together + bool connect_break_targets(); //connecting break instructions with proper targets - // iterate across m_basic_blocks of function, - // finding postdominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - void find_postdominators(); - void print_postdominators(); + //iterate across m_basic_blocks of function, + //finding dominator blocks, using algorithm of + //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 + void find_dominators( ); + void print_dominators(); + void find_idominators(); + void print_idominators(); - // iterate across m_basic_blocks of function, - // finding immediate postdominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - void find_ipostdominators(); - void print_ipostdominators(); - void do_pdom(); // function to call pdom analysis + //iterate across m_basic_blocks of function, + //finding postdominator blocks, using algorithm of + //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 + void find_postdominators( ); + void print_postdominators(); - unsigned get_num_reconvergence_pairs(); + //iterate across m_basic_blocks of function, + //finding immediate postdominator blocks, using algorithm of + //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + void find_ipostdominators( ); + void print_ipostdominators(); + void do_pdom(); //function to call pdom analysis - void get_reconvergence_pairs(gpgpu_recon_t *recon_points); + unsigned get_num_reconvergence_pairs(); - unsigned get_function_size() { return m_instructions.size(); } + void get_reconvergence_pairs(gpgpu_recon_t* recon_points); - void ptx_assemble(); + unsigned get_function_size() { return m_instructions.size();} - unsigned ptx_get_inst_op(ptx_thread_info *thread); - void add_param(const char *name, struct param_t value) { - m_kernel_params[name] = value; - } - void add_param_name_type_size(unsigned index, std::string name, int type, - size_t size, bool ptr, memory_space_t space); - void add_param_data(unsigned argn, struct gpgpu_ptx_sim_arg *args); - void add_return_var(const symbol *rv) { m_return_var_sym = rv; } - void add_arg(const symbol *arg) { - assert(arg != NULL); - m_args.push_back(arg); - } - void remove_args() { m_args.clear(); } - unsigned num_args() const { return m_args.size(); } - unsigned get_args_aligned_size(); + void ptx_assemble(); + + unsigned ptx_get_inst_op( ptx_thread_info *thread ); + void add_param( const char *name, struct param_t value ) + { + m_kernel_params[ name ] = value; + } + void add_param_name_type_size( unsigned index, std::string name, int type, size_t size, bool ptr, memory_space_t space ); + void add_param_data( unsigned argn, struct gpgpu_ptx_sim_arg *args ); + void add_return_var( const symbol *rv ) + { + m_return_var_sym = rv; + } + void add_arg( const symbol *arg ) + { + assert( arg != NULL ); + m_args.push_back(arg); + } + void remove_args() + { + m_args.clear(); + } + unsigned num_args() const + { + return m_args.size(); + } + unsigned get_args_aligned_size(); - const symbol *get_arg(unsigned n) const { - assert(n < m_args.size()); - return m_args[n]; - } - bool has_return() const { return m_return_var_sym != NULL; } - const symbol *get_return_var() const { return m_return_var_sym; } - const ptx_instruction *get_instruction(unsigned PC) const { - unsigned index = PC - m_start_PC; - if (index < m_instr_mem_size) return m_instr_mem[index]; - return NULL; - } - addr_t get_start_PC() const { return m_start_PC; } + const symbol* get_arg( unsigned n ) const + { + assert( n < m_args.size() ); + return m_args[n]; + } + bool has_return() const + { + return m_return_var_sym != NULL; + } + const symbol *get_return_var() const + { + return m_return_var_sym; + } + const ptx_instruction *get_instruction( unsigned PC ) const + { + unsigned index = PC - m_start_PC; + if( index < m_instr_mem_size ) + return m_instr_mem[index]; + return NULL; + } + addr_t get_start_PC() const + { + return m_start_PC; + } - void finalize(memory_space *param_mem); - void param_to_shared(memory_space *shared_mem, symbol_table *symtab); - void list_param(FILE *fout) const; - void ptx_jit_config(std::map<unsigned long long, size_t> mallocPtr_Size, - memory_space *param_mem, gpgpu_t *gpu, dim3 gridDim, - dim3 blockDim); + void finalize( memory_space *param_mem ); + void param_to_shared( memory_space *shared_mem, symbol_table *symtab ); + void list_param( FILE *fout ) const; + void ptx_jit_config(std::map<unsigned long long, size_t> mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) ; - const struct gpgpu_ptx_sim_info *get_kernel_info() const { - assert(m_kernel_info.maxthreads == maxnt_id); - return &m_kernel_info; - } + const struct gpgpu_ptx_sim_info* get_kernel_info () const + { + assert (m_kernel_info.maxthreads == maxnt_id); + return &m_kernel_info; + } - const void set_kernel_info(const struct gpgpu_ptx_sim_info &info) { - m_kernel_info = info; - m_kernel_info.ptx_version = 10 * get_ptx_version().ver(); - m_kernel_info.sm_target = get_ptx_version().target(); - // THIS DEPENDS ON ptxas being called after the PTX is parsed. - m_kernel_info.maxthreads = maxnt_id; - } - symbol_table *get_symtab() { return m_symtab; } + const void set_kernel_info (const struct gpgpu_ptx_sim_info &info) { + m_kernel_info = info; + m_kernel_info.ptx_version = 10*get_ptx_version().ver(); + m_kernel_info.sm_target = get_ptx_version().target(); + // THIS DEPENDS ON ptxas being called after the PTX is parsed. + m_kernel_info.maxthreads = maxnt_id; + } + symbol_table *get_symtab() + { + return m_symtab; + } - unsigned local_mem_framesize() const { return m_local_mem_framesize; } - void set_framesize(unsigned sz) { m_local_mem_framesize = sz; } - bool is_entry_point() const { return m_entry_point; } - bool is_pdom_set() const { return pdom_done; } // return pdom flag - void set_pdom() { pdom_done = true; } // set pdom flag + unsigned local_mem_framesize() const + { + return m_local_mem_framesize; + } + void set_framesize( unsigned sz ) + { + m_local_mem_framesize = sz; + } + bool is_entry_point() const { return m_entry_point; } + bool is_pdom_set() const { return pdom_done; } //return pdom flag + void set_pdom() { pdom_done = true; } //set pdom flag - void add_config_param(size_t size, unsigned alignment) { - unsigned offset = 0; - if (m_param_configs.size() > 0) { - unsigned offset_nom = - m_param_configs.back().first + m_param_configs.back().second; - // ensure offset matches alignment requirements - offset = offset_nom % alignment ? (offset_nom / alignment + 1) * alignment - : offset_nom; - } - m_param_configs.push_back(std::pair<size_t, unsigned>(size, offset)); - } - - std::pair<size_t, unsigned> get_param_config(unsigned param_num) const { - return m_param_configs[param_num]; - } + void add_config_param( size_t size, unsigned alignment ){ + unsigned offset = 0; + if (m_param_configs.size()>0){ + unsigned offset_nom = m_param_configs.back().first + m_param_configs.back().second; + //ensure offset matches alignment requirements + offset = offset_nom%alignment ? (offset_nom/alignment + 1) * alignment : offset_nom; + } + m_param_configs.push_back(std::pair<size_t,unsigned>(size, offset)); + } - void set_maxnt_id(unsigned maxthreads) { maxnt_id = maxthreads; } - unsigned get_maxnt_id() { return maxnt_id; } - // backward pointer - class gpgpu_context *gpgpu_ctx; + std::pair<size_t, unsigned> get_param_config(unsigned param_num) const { return m_param_configs[param_num]; } - private: - unsigned maxnt_id; - unsigned m_uid; - unsigned m_local_mem_framesize; - bool m_entry_point; - bool m_extern; - bool m_assembled; - bool pdom_done; // flag to check whether pdom is completed or not - std::string m_name; - ptx_instruction **m_instr_mem; - unsigned m_start_PC; - unsigned m_instr_mem_size; - std::map<std::string, param_t> m_kernel_params; - std::map<unsigned, param_info> m_ptx_kernel_param_info; - std::vector<std::pair<size_t, unsigned> > m_param_configs; - const symbol *m_return_var_sym; - std::vector<const symbol *> m_args; - std::list<ptx_instruction *> m_instructions; - std::vector<basic_block_t *> m_basic_blocks; - std::list<std::pair<unsigned, unsigned> > m_back_edges; - std::map<std::string, unsigned> labels; - unsigned num_reconvergence_pairs; + void set_maxnt_id(unsigned maxthreads) { maxnt_id = maxthreads;} + unsigned get_maxnt_id() { return maxnt_id;} + // backward pointer + class gpgpu_context* gpgpu_ctx; - // Registers/shmem/etc. used (from ptxas -v), loaded from ___.ptxinfo along - // with ___.ptx - struct gpgpu_ptx_sim_info m_kernel_info; +private: + unsigned maxnt_id; + unsigned m_uid; + unsigned m_local_mem_framesize; + bool m_entry_point; + bool m_extern; + bool m_assembled; + bool pdom_done; //flag to check whether pdom is completed or not + std::string m_name; + ptx_instruction **m_instr_mem; + unsigned m_start_PC; + unsigned m_instr_mem_size; + std::map<std::string,param_t> m_kernel_params; + std::map<unsigned,param_info> m_ptx_kernel_param_info; + std::vector< std::pair<size_t, unsigned> > m_param_configs; + const symbol *m_return_var_sym; + std::vector<const symbol*> m_args; + std::list<ptx_instruction*> m_instructions; + std::vector<basic_block_t*> m_basic_blocks; + std::list<std::pair<unsigned, unsigned> > m_back_edges; + std::map<std::string,unsigned> labels; + unsigned num_reconvergence_pairs; - symbol_table *m_symtab; + //Registers/shmem/etc. used (from ptxas -v), loaded from ___.ptxinfo along with ___.ptx + struct gpgpu_ptx_sim_info m_kernel_info; - // parameter size for device kernels - int m_args_aligned_size; + symbol_table *m_symtab; - addr_t m_n; // offset in m_instr_mem (used in do_pdom) + //parameter size for device kernels + int m_args_aligned_size; + + addr_t m_n; // offset in m_instr_mem (used in do_pdom) }; class arg_buffer_t { - public: - arg_buffer_t(gpgpu_context *ctx) : m_src_op(ctx) { - m_is_reg = false; - m_is_param = false; - m_param_value = NULL; - m_reg_value = ptx_reg_t(); - } - arg_buffer_t(const arg_buffer_t &another, gpgpu_context *ctx) - : m_src_op(ctx) { - make_copy(another); - } - void make_copy(const arg_buffer_t &another) { - m_dst = another.m_dst; - m_src_op = another.m_src_op; - m_is_reg = another.m_is_reg; - m_is_param = another.m_is_param; - m_reg_value = another.m_reg_value; - m_param_bytes = another.m_param_bytes; - if (m_is_param) { - m_param_value = malloc(m_param_bytes); - memcpy(m_param_value, another.m_param_value, m_param_bytes); - } - } - void operator=(const arg_buffer_t &another) { make_copy(another); } - ~arg_buffer_t() { - if (m_is_param) free(m_param_value); - } - arg_buffer_t(const symbol *dst_sym, const operand_info &src_op, - ptx_reg_t source_value) - : m_src_op(src_op) { - m_dst = dst_sym; - m_reg_value = ptx_reg_t(); - if (dst_sym->is_reg()) { - m_is_reg = true; - m_is_param = false; - assert(src_op.is_reg()); - m_reg_value = source_value; - } else { - m_is_param = true; - m_is_reg = false; - m_param_value = calloc(sizeof(ptx_reg_t), 1); - // new (m_param_value) ptx_reg_t(source_value); - memcpy(m_param_value, &source_value, sizeof(ptx_reg_t)); - m_param_bytes = sizeof(ptx_reg_t); - } - } - arg_buffer_t(const symbol *dst_sym, const operand_info &src_op, - void *source_param_value_array, unsigned array_size) - : m_src_op(src_op) { - m_dst = dst_sym; - if (dst_sym->is_reg()) { - m_is_reg = true; - m_is_param = false; - assert(src_op.is_param_local()); - assert(dst_sym->get_size_in_bytes() == array_size); - switch (array_size) { - case 1: - m_reg_value.u8 = *(unsigned char *)source_param_value_array; - break; - case 2: - m_reg_value.u16 = *(unsigned short *)source_param_value_array; - break; - case 4: - m_reg_value.u32 = *(unsigned int *)source_param_value_array; - break; - case 8: - m_reg_value.u64 = *(unsigned long long *)source_param_value_array; - break; - default: - printf( - "GPGPU-Sim PTX: ERROR ** source param size does not match known " - "register sizes\n"); - break; +public: + arg_buffer_t(gpgpu_context* ctx) : m_src_op(ctx) + { + m_is_reg=false; + m_is_param=false; + m_param_value=NULL; + m_reg_value=ptx_reg_t(); + } + arg_buffer_t( const arg_buffer_t &another, gpgpu_context* ctx ) : m_src_op(ctx) + { + make_copy(another); + } + void make_copy( const arg_buffer_t &another ) + { + m_dst = another.m_dst; + m_src_op = another.m_src_op; + m_is_reg = another.m_is_reg; + m_is_param = another.m_is_param; + m_reg_value = another.m_reg_value; + m_param_bytes = another.m_param_bytes; + if( m_is_param ) { + m_param_value = malloc(m_param_bytes); + memcpy(m_param_value,another.m_param_value,m_param_bytes); } - } else { - // param - m_is_param = true; - m_is_reg = false; - m_param_value = calloc(array_size, 1); - m_param_bytes = array_size; - memcpy(m_param_value, source_param_value_array, array_size); - } - } + } + void operator=( const arg_buffer_t &another ) + { + make_copy(another); + } + ~arg_buffer_t() + { + if( m_is_param ) + free(m_param_value); + } + arg_buffer_t( const symbol *dst_sym, const operand_info &src_op, ptx_reg_t source_value ) : m_src_op(src_op) + { + m_dst = dst_sym; + m_reg_value=ptx_reg_t(); + if( dst_sym->is_reg() ) { + m_is_reg = true; + m_is_param = false; + assert( src_op.is_reg() ); + m_reg_value = source_value; + } else { + m_is_param = true; + m_is_reg = false; + m_param_value = calloc(sizeof(ptx_reg_t),1); + //new (m_param_value) ptx_reg_t(source_value); + memcpy(m_param_value,&source_value,sizeof(ptx_reg_t)); + m_param_bytes = sizeof(ptx_reg_t); + } + } + arg_buffer_t( const symbol *dst_sym, const operand_info &src_op, void *source_param_value_array, unsigned array_size ) : m_src_op(src_op) + { + m_dst = dst_sym; + if( dst_sym->is_reg() ) { + m_is_reg = true; + m_is_param = false; + assert( src_op.is_param_local() ); + assert( dst_sym->get_size_in_bytes() == array_size ); + switch( array_size ) { + case 1: m_reg_value.u8 = *(unsigned char*)source_param_value_array; break; + case 2: m_reg_value.u16 = *(unsigned short*)source_param_value_array; break; + case 4: m_reg_value.u32 = *(unsigned int*)source_param_value_array; break; + case 8: m_reg_value.u64 = *(unsigned long long*)source_param_value_array; break; + default: + printf("GPGPU-Sim PTX: ERROR ** source param size does not match known register sizes\n"); + break; + } + } else { + // param + m_is_param = true; + m_is_reg = false; + m_param_value = calloc(array_size,1); + m_param_bytes = array_size; + memcpy(m_param_value,source_param_value_array,array_size); + } + } - bool is_reg() const { return m_is_reg; } - ptx_reg_t get_reg() const { - assert(m_is_reg); - return m_reg_value; - } + bool is_reg() const { return m_is_reg; } + ptx_reg_t get_reg() const + { + assert(m_is_reg); + return m_reg_value; + } - const void *get_param_buffer() const { - assert(m_is_param); - return m_param_value; - } - size_t get_param_buffer_size() const { - assert(m_is_param); - return m_param_bytes; - } + const void *get_param_buffer() const + { + assert(m_is_param); + return m_param_value; + } + size_t get_param_buffer_size() const + { + assert(m_is_param); + return m_param_bytes; + } - const symbol *get_dst() const { return m_dst; } + const symbol *get_dst() const { return m_dst; } - private: - // destination of copy - const symbol *m_dst; +private: + // destination of copy + const symbol *m_dst; - // source operand - operand_info m_src_op; + // source operand + operand_info m_src_op; - // source information - bool m_is_reg; - bool m_is_param; + // source information + bool m_is_reg; + bool m_is_param; - // source is register - ptx_reg_t m_reg_value; + // source is register + ptx_reg_t m_reg_value; - // source is param - void *m_param_value; - unsigned m_param_bytes; + // source is param + void *m_param_value; + unsigned m_param_bytes; }; -typedef std::list<arg_buffer_t> arg_buffer_list_t; -arg_buffer_t copy_arg_to_buffer(ptx_thread_info *thread, - operand_info actual_param_op, - const symbol *formal_param); -void copy_args_into_buffer_list(const ptx_instruction *pI, - ptx_thread_info *thread, - const function_info *target_func, - arg_buffer_list_t &arg_values); -void copy_buffer_list_into_frame(ptx_thread_info *thread, - arg_buffer_list_t &arg_values); -void copy_buffer_to_frame(ptx_thread_info *thread, const arg_buffer_t &a); +typedef std::list< arg_buffer_t > arg_buffer_list_t; +arg_buffer_t copy_arg_to_buffer(ptx_thread_info * thread, operand_info actual_param_op, const symbol * formal_param); +void copy_args_into_buffer_list( const ptx_instruction * pI, + ptx_thread_info * thread, + const function_info * target_func, + arg_buffer_list_t &arg_values ); +void copy_buffer_list_into_frame(ptx_thread_info * thread, arg_buffer_list_t &arg_values); +void copy_buffer_to_frame(ptx_thread_info * thread, const arg_buffer_t &a); + struct textureInfo { - unsigned int texel_size; // size in bytes, e.g. (channelDesc.x+y+z+w)/8 - unsigned int Tx, - Ty; // tiling factor dimensions of layout of texels per 64B cache block - unsigned int Tx_numbits, Ty_numbits; // log2(T) - unsigned int texel_size_numbits; // log2(texel_size) + unsigned int texel_size; //size in bytes, e.g. (channelDesc.x+y+z+w)/8 + unsigned int Tx,Ty; //tiling factor dimensions of layout of texels per 64B cache block + unsigned int Tx_numbits,Ty_numbits; //log2(T) + unsigned int texel_size_numbits; //log2(texel_size) }; -extern std::map<std::string, symbol_table *> g_sym_name_to_symbol_table; +extern std::map<std::string,symbol_table*> g_sym_name_to_symbol_table; + -void gpgpu_ptx_assemble(std::string kname, void *kinfo); +void gpgpu_ptx_assemble( std::string kname, void *kinfo ); #include "../option_parser.h" -unsigned ptx_kernel_shmem_size(void *kernel_impl); -unsigned ptx_kernel_nregs(void *kernel_impl); +unsigned ptx_kernel_shmem_size( void *kernel_impl ); +unsigned ptx_kernel_nregs( void *kernel_impl ); #endif diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc index af66314..33bcf45 100644 --- a/src/cuda-sim/ptx_loader.cc +++ b/src/cuda-sim/ptx_loader.cc @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -28,556 +26,503 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "ptx_loader.h" -#include <dirent.h> +#include "ptx_ir.h" +#include "cuda-sim.h" +#include "ptx_parser.h" #include <unistd.h> +#include <dirent.h> #include <fstream> #include <sstream> #include "../../libcuda/gpgpu_context.h" -#include "cuda-sim.h" -#include "ptx_ir.h" -#include "ptx_parser.h" /// extern prototypes -extern int ptx_error(yyscan_t yyscanner, const char *s); -extern int ptx_lex_init(yyscan_t *scanner); -extern void ptx_set_in(FILE *_in_str, yyscan_t yyscanner); -extern int ptx_parse(yyscan_t scanner, ptx_recognizer *recognizer); +extern int ptx_error( yyscan_t yyscanner, const char *s ); +extern int ptx_lex_init(yyscan_t* scanner); +extern void ptx_set_in(FILE * _in_str ,yyscan_t yyscanner ); +extern int ptx_parse(yyscan_t scanner, ptx_recognizer* recognizer); extern int ptx_lex_destroy(yyscan_t scanner); -extern int ptx__scan_string(const char *, yyscan_t scanner); +extern int ptx__scan_string(const char*, yyscan_t scanner); -extern std::map<unsigned, const char *> get_duplicate(); +extern std::map<unsigned,const char*> get_duplicate(); -typedef void *yyscan_t; -extern int ptxinfo_lex_init(yyscan_t *scanner); -extern void ptxinfo_set_in(FILE *_in_str, yyscan_t yyscanner); -extern int ptxinfo_parse(yyscan_t scanner, ptxinfo_data *ptxinfo); +typedef void * yyscan_t; +extern int ptxinfo_lex_init(yyscan_t* scanner); +extern void ptxinfo_set_in (FILE * _in_str ,yyscan_t yyscanner ); +extern int ptxinfo_parse(yyscan_t scanner, ptxinfo_data* ptxinfo); extern int ptxinfo_lex_destroy(yyscan_t scanner); static bool g_save_embedded_ptx; static int g_occupancy_sm_number; -bool ptxinfo_data::keep_intermediate_files() { - return g_keep_intermediate_files; -} +bool ptxinfo_data::keep_intermediate_files() {return g_keep_intermediate_files;} -void gpgpu_context::ptx_reg_options(option_parser_t opp) { - option_parser_register(opp, "-save_embedded_ptx", OPT_BOOL, - &g_save_embedded_ptx, - "saves ptx files embedded in binary as <n>.ptx", "0"); - option_parser_register(opp, "-keep", OPT_BOOL, - &(ptxinfo->g_keep_intermediate_files), - "keep intermediate files created by GPGPU-Sim when " - "interfacing with external programs", - "0"); - option_parser_register(opp, "-gpgpu_ptx_save_converted_ptxplus", OPT_BOOL, - &(ptxinfo->m_ptx_save_converted_ptxplus), - "Saved converted ptxplus to a file", "0"); - option_parser_register(opp, "-gpgpu_occupancy_sm_number", OPT_INT32, - &g_occupancy_sm_number, - "The SM number to pass to ptxas when getting register " - "usage for computing GPU occupancy. " - "This parameter is required in the config.", - "0"); +void gpgpu_context::ptx_reg_options(option_parser_t opp) +{ + option_parser_register(opp, "-save_embedded_ptx", OPT_BOOL, &g_save_embedded_ptx, + "saves ptx files embedded in binary as <n>.ptx", + "0"); + option_parser_register(opp, "-keep", OPT_BOOL, &(ptxinfo->g_keep_intermediate_files), + "keep intermediate files created by GPGPU-Sim when interfacing with external programs", + "0"); + option_parser_register(opp, "-gpgpu_ptx_save_converted_ptxplus", OPT_BOOL, + &(ptxinfo->m_ptx_save_converted_ptxplus), + "Saved converted ptxplus to a file", + "0"); + option_parser_register(opp, "-gpgpu_occupancy_sm_number", OPT_INT32, &g_occupancy_sm_number, + "The SM number to pass to ptxas when getting register usage for computing GPU occupancy. " + "This parameter is required in the config.", + "0"); } -void gpgpu_context::print_ptx_file(const char *p, unsigned source_num, - const char *filename) { - printf("\nGPGPU-Sim PTX: file _%u.ptx contents:\n\n", source_num); - char *s = strdup(p); - char *t = s; - unsigned n = 1; - while (*t != '\0') { - char *u = t; - while ((*u != '\n') && (*u != '\0')) u++; - unsigned last = (*u == '\0'); - *u = '\0'; - const ptx_instruction *pI = ptx_parser->ptx_instruction_lookup(filename, n); - char pc[64]; - if (pI && pI->get_PC()) - snprintf(pc, 64, "%4u", pI->get_PC()); - else - snprintf(pc, 64, " "); - printf(" _%u.ptx %4u (pc=%s): %s\n", source_num, n, pc, t); - if (last) break; - t = u + 1; - n++; - } - free(s); - fflush(stdout); +void gpgpu_context::print_ptx_file( const char *p, unsigned source_num, const char *filename ) +{ + printf("\nGPGPU-Sim PTX: file _%u.ptx contents:\n\n", source_num ); + char *s = strdup(p); + char *t = s; + unsigned n=1; + while ( *t != '\0' ) { + char *u = t; + while ( (*u != '\n') && (*u != '\0') ) u++; + unsigned last = (*u == '\0'); + *u = '\0'; + const ptx_instruction *pI = ptx_parser->ptx_instruction_lookup(filename,n); + char pc[64]; + if( pI && pI->get_PC() ) + snprintf(pc,64,"%4u", pI->get_PC() ); + else + snprintf(pc,64," "); + printf(" _%u.ptx %4u (pc=%s): %s\n", source_num, n, pc, t ); + if ( last ) break; + t = u+1; + n++; + } + free(s); + fflush(stdout); } -char *ptxinfo_data::gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus( - const std::string ptxfilename, const std::string elffilename, - const std::string sassfilename) { - printf("GPGPU-Sim PTX: converting EMBEDDED .ptx file to ptxplus \n"); +char* ptxinfo_data::gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptxfilename, const std::string elffilename, const std::string sassfilename) +{ - char fname_ptxplus[1024]; - snprintf(fname_ptxplus, 1024, "_ptxplus_XXXXXX"); - int fd4 = mkstemp(fname_ptxplus); - close(fd4); + printf("GPGPU-Sim PTX: converting EMBEDDED .ptx file to ptxplus \n"); - // Run cuobjdump_to_ptxplus - char commandline[1024]; - int result; - snprintf(commandline, 1024, - "$GPGPUSIM_ROOT/build/$GPGPUSIM_CONFIG/cuobjdump_to_ptxplus/" - "cuobjdump_to_ptxplus %s %s %s %s", - ptxfilename.c_str(), sassfilename.c_str(), elffilename.c_str(), - fname_ptxplus); - fflush(stdout); - printf("GPGPU-Sim PTX: calling cuobjdump_to_ptxplus\ncommandline: %s\n", - commandline); - result = system(commandline); - if (result) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** could not execute %s\n", - commandline); - exit(1); - } + char fname_ptxplus[1024]; + snprintf(fname_ptxplus,1024,"_ptxplus_XXXXXX"); + int fd4=mkstemp(fname_ptxplus); + close(fd4); - // Get ptxplus from file - std::ifstream fileStream(fname_ptxplus, std::ios::in); - std::string text, line; - while (getline(fileStream, line)) { - text += (line + "\n"); - } - fileStream.close(); + // Run cuobjdump_to_ptxplus + char commandline[1024]; + int result; + snprintf(commandline, 1024, "$GPGPUSIM_ROOT/build/$GPGPUSIM_CONFIG/cuobjdump_to_ptxplus/cuobjdump_to_ptxplus %s %s %s %s", + ptxfilename.c_str(), + sassfilename.c_str(), + elffilename.c_str(), + fname_ptxplus); + fflush(stdout); + printf("GPGPU-Sim PTX: calling cuobjdump_to_ptxplus\ncommandline: %s\n", commandline); + result = system(commandline); + if(result){fprintf(stderr, "GPGPU-Sim PTX: ERROR ** could not execute %s\n", commandline); exit(1);} - char *ptxplus_str = new char[strlen(text.c_str()) + 1]; - strcpy(ptxplus_str, text.c_str()); - if (!m_ptx_save_converted_ptxplus) { - char rm_commandline[1024]; + // Get ptxplus from file + std::ifstream fileStream(fname_ptxplus, std::ios::in); + std::string text, line; + while(getline(fileStream,line)) { + text += (line + "\n"); + } + fileStream.close(); - snprintf(rm_commandline, 1024, "rm -f %s", fname_ptxplus); + char* ptxplus_str = new char [strlen(text.c_str())+1]; + strcpy(ptxplus_str, text.c_str()); - printf("GPGPU-Sim PTX: removing temporary files using \"%s\"\n", - rm_commandline); - int rm_result = system(rm_commandline); - if (rm_result != 0) { - fprintf(stderr, - "GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", - rm_result); - exit(1); - } - } - printf("GPGPU-Sim PTX: DONE converting EMBEDDED .ptx file to ptxplus \n"); + if (!m_ptx_save_converted_ptxplus){ + char rm_commandline[1024]; - return ptxplus_str; + snprintf(rm_commandline,1024,"rm -f %s", fname_ptxplus); + + printf("GPGPU-Sim PTX: removing temporary files using \"%s\"\n", rm_commandline); + int rm_result = system(rm_commandline); + if( rm_result != 0 ) { + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", rm_result); + exit(1); + } + } + printf("GPGPU-Sim PTX: DONE converting EMBEDDED .ptx file to ptxplus \n"); + + return ptxplus_str; } -symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_string( - const char *p, unsigned source_num) { - char buf[1024]; - snprintf(buf, 1024, "_%u.ptx", source_num); - if (g_save_embedded_ptx) { - FILE *fp = fopen(buf, "w"); - fprintf(fp, "%s", p); - fclose(fp); - } - symbol_table *symtab = init_parser(buf); - ptx_lex_init(&(ptx_parser->scanner)); - ptx__scan_string(p, ptx_parser->scanner); - int errors = ptx_parse(ptx_parser->scanner, ptx_parser); - if (errors) { - char fname[1024]; - snprintf(fname, 1024, "_ptx_errors_XXXXXX"); - int fd = mkstemp(fname); - close(fd); - printf( - "GPGPU-Sim PTX: parser error detected, exiting... but first extracting " - ".ptx to \"%s\"\n", - fname); - FILE *ptxfile = fopen(fname, "w"); - fprintf(ptxfile, "%s", p); - fclose(ptxfile); - abort(); - exit(40); - } - ptx_lex_destroy(ptx_parser->scanner); - if (g_debug_execution >= 100) print_ptx_file(p, source_num, buf); +symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_string( const char *p, unsigned source_num ) +{ + char buf[1024]; + snprintf(buf,1024,"_%u.ptx", source_num ); + if( g_save_embedded_ptx ) { + FILE *fp = fopen(buf,"w"); + fprintf(fp,"%s",p); + fclose(fp); + } + symbol_table *symtab=init_parser(buf); + ptx_lex_init(&(ptx_parser->scanner)); + ptx__scan_string(p, ptx_parser->scanner); + int errors = ptx_parse (ptx_parser->scanner, ptx_parser); + if ( errors ) { + char fname[1024]; + snprintf(fname,1024,"_ptx_errors_XXXXXX"); + int fd=mkstemp(fname); + close(fd); + printf("GPGPU-Sim PTX: parser error detected, exiting... but first extracting .ptx to \"%s\"\n", fname); + FILE *ptxfile = fopen(fname,"w"); + fprintf(ptxfile,"%s", p ); + fclose(ptxfile); + abort(); + exit(40); + } + ptx_lex_destroy(ptx_parser->scanner); + + if ( g_debug_execution >= 100 ) + print_ptx_file(p,source_num,buf); - printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n", buf); - return symtab; + printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n",buf); + return symtab; } -symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_filename( - const char *filename) { - symbol_table *symtab = init_parser(filename); - printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n", filename); - return symtab; +symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_filename( const char *filename ) +{ + symbol_table *symtab=init_parser(filename); + printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n",filename); + return symtab; } void fix_duplicate_errors(char fname2[1024]) { - char tempfile[1024] = "_temp_ptx"; - char commandline[1024]; - - // change the name of the ptx file to _temp_ptx - snprintf(commandline, 1024, "mv %s %s", fname2, tempfile); - printf("Running: %s\n", commandline); - int result = system(commandline); - if (result != 0) { - fprintf(stderr, - "GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", - fname2, tempfile); - exit(1); - } + char tempfile[1024] = "_temp_ptx"; + char commandline[1024]; - // store all of the ptx into a char array - FILE *ptxsource = fopen(tempfile, "r"); - fseek(ptxsource, 0, SEEK_END); - long filesize = ftell(ptxsource); - rewind(ptxsource); - char *ptxdata = (char *)malloc((filesize + 1) * sizeof(char)); - // Fail if we do not read the file - assert(fread(ptxdata, filesize, 1, ptxsource) == 1); - fclose(ptxsource); + // change the name of the ptx file to _temp_ptx + snprintf(commandline,1024,"mv %s %s",fname2,tempfile); + printf("Running: %s\n", commandline); + int result = system(commandline); + if (result != 0) { + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", fname2, tempfile); + exit(1); + } - FILE *ptxdest = fopen(fname2, "w"); - std::map<unsigned, const char *> duplicate = get_duplicate(); - unsigned offset; - unsigned oldlinenum = 1; - unsigned linenum; - char *startptr = ptxdata; - char *funcptr; - char *tempptr = ptxdata - 1; - char *lineptr = ptxdata - 1; + // store all of the ptx into a char array + FILE *ptxsource = fopen(tempfile,"r"); + fseek(ptxsource, 0, SEEK_END); + long filesize = ftell(ptxsource); + rewind(ptxsource); + char *ptxdata = (char*)malloc((filesize+1)*sizeof(char)); + // Fail if we do not read the file + assert(fread(ptxdata, filesize, 1, ptxsource) == 1); + fclose(ptxsource); - // recreate the ptx file without duplications - for (std::map<unsigned, const char *>::iterator iter = duplicate.begin(); - iter != duplicate.end(); iter++) { - // find the line of the next error - linenum = iter->first; - for (int i = oldlinenum; i < linenum; i++) { - lineptr = strchr(lineptr + 1, '\n'); - } + FILE *ptxdest = fopen(fname2,"w"); + std::map<unsigned,const char*> duplicate = get_duplicate(); + unsigned offset; + unsigned oldlinenum = 1; + unsigned linenum; + char *startptr = ptxdata; + char *funcptr; + char *tempptr = ptxdata - 1; + char *lineptr = ptxdata - 1; - // find the end of the current section to be copied over - // then find the start of the next section that will be copied - if (strcmp("function", iter->second) == 0) { - // get location of most recent .func - while (tempptr < lineptr && tempptr != NULL) { - funcptr = tempptr; - tempptr = strstr(funcptr + 1, ".func"); - } + // recreate the ptx file without duplications + for ( std::map<unsigned,const char*>::iterator iter = duplicate.begin(); + iter != duplicate.end(); + iter++){ + // find the line of the next error + linenum = iter->first; + for (int i = oldlinenum; i < linenum; i++) { + lineptr = strchr(lineptr + 1, '\n'); + } + + // find the end of the current section to be copied over + // then find the start of the next section that will be copied + if (strcmp("function", iter->second) == 0) { + // get location of most recent .func + while (tempptr < lineptr && tempptr != NULL) { + funcptr = tempptr; + tempptr = strstr(funcptr + 1, ".func"); + } - // get the start of the previous line - offset = 0; - while (*(funcptr - offset) != '\n') offset++; + // get the start of the previous line + offset = 0; + while (*(funcptr - offset) != '\n') offset++; - fwrite(startptr, sizeof(char), funcptr - offset + 1 - startptr, ptxdest); - - // find next location of startptr - if (*(lineptr + 3) == ';') { - // for function definitions - startptr = lineptr + 5; - } else if (*(lineptr + 3) == '{') { - // for functions enclosed with curly brackets - offset = 5; - unsigned bracket = 1; - while (bracket != 0) { - if (*(lineptr + offset) == '{') - bracket++; - else if (*(lineptr + offset) == '}') - bracket--; - offset++; - } - startptr = lineptr + offset + 1; - } else { - printf("GPGPU-Sim PTX: ERROR ** Unrecognized function format\n"); - abort(); - } - } else if (strcmp("variable", iter->second) == 0) { - fwrite(startptr, sizeof(char), (int)(lineptr + 1 - startptr), ptxdest); + fwrite(startptr, sizeof(char), funcptr - offset + 1 - startptr, ptxdest); - // find next location of startptr - offset = 1; - while (*(lineptr + offset) != '\n') offset++; - startptr = lineptr + offset + 1; - } else { - printf("GPGPU-Sim PTX: ERROR ** Unsupported duplicate type: %s\n", - iter->second); - } + //find next location of startptr + if (*(lineptr + 3) == ';') { + // for function definitions + startptr = lineptr + 5; + } else if (*(lineptr + 3) == '{') { + // for functions enclosed with curly brackets + offset = 5; + unsigned bracket = 1; + while (bracket != 0) { + if (*(lineptr + offset) == '{') bracket++; + else if (*(lineptr + offset) == '}') bracket--; + offset++; + } + startptr = lineptr + offset + 1; + } else { + printf("GPGPU-Sim PTX: ERROR ** Unrecognized function format\n"); + abort(); + } + } else if (strcmp("variable", iter->second) == 0) { + fwrite(startptr, sizeof(char), (int)(lineptr + 1 - startptr), ptxdest); + + //find next location of startptr + offset = 1; + while (*(lineptr + offset) != '\n') offset++; + startptr = lineptr + offset + 1; + } else { + printf("GPGPU-Sim PTX: ERROR ** Unsupported duplicate type: %s\n", iter->second); + } - oldlinenum = linenum; - } - // copy over the rest of the file - fwrite(startptr, sizeof(char), ptxdata + filesize - startptr, ptxdest); + oldlinenum = linenum; + } + // copy over the rest of the file + fwrite(startptr, sizeof(char), ptxdata + filesize - startptr, ptxdest); - // cleanup - free(ptxdata); - fclose(ptxdest); - snprintf(commandline, 1024, "rm -f %s", tempfile); - printf("Running: %s\n", commandline); - result = system(commandline); - if (result != 0) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile); - exit(1); - } + // cleanup + free(ptxdata); + fclose(ptxdest); + snprintf(commandline,1024,"rm -f %s",tempfile); + printf("Running: %s\n", commandline); + result = system(commandline); + if (result != 0) { + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile); + exit(1); + } } -// we need the application name here too. -char *get_app_binary_name() { - char exe_path[1025]; - char *self_exe_path; + +//we need the application name here too. +char* get_app_binary_name(){ + char exe_path[1025]; + char *self_exe_path; #ifdef __APPLE__ - // AMRUTH: get apple device and check the result. - printf("WARNING: not tested for Apple-mac devices \n"); - abort(); + //AMRUTH: get apple device and check the result. + printf("WARNING: not tested for Apple-mac devices \n"); + abort(); #else - std::stringstream exec_link; - exec_link << "/proc/self/exe"; - ssize_t path_length = readlink(exec_link.str().c_str(), exe_path, 1024); - assert(path_length != -1); - exe_path[path_length] = '\0'; + std::stringstream exec_link; + exec_link << "/proc/self/exe"; + ssize_t path_length = readlink(exec_link.str().c_str(), exe_path, 1024); + assert(path_length != -1); + exe_path[path_length] = '\0'; - char *token = strtok(exe_path, "/"); - while (token != NULL) { - self_exe_path = token; - token = strtok(NULL, "/"); - } + char *token = strtok(exe_path, "/"); + while(token !=NULL){ + self_exe_path = token; + token = strtok(NULL,"/"); + } #endif - self_exe_path = strtok(self_exe_path, "."); - printf("self exe links to: %s\n", self_exe_path); - return self_exe_path; + self_exe_path = strtok(self_exe_path, "."); + printf("self exe links to: %s\n", self_exe_path); + return self_exe_path; } -void gpgpu_context::gpgpu_ptx_info_load_from_filename(const char *filename, - unsigned sm_version) { - std::string ptxas_filename(std::string(filename) + "as"); - char buff[1024], extra_flags[1024]; - extra_flags[0] = 0; - if (!device_runtime->g_cdp_enabled) - snprintf(extra_flags, 1024, "--gpu-name=sm_%u", sm_version); - else - snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", sm_version); - snprintf( - buff, 1024, - "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", - extra_flags, filename, ptxas_filename.c_str()); - int result = system(buff); - if (result != 0) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); - } +void gpgpu_context::gpgpu_ptx_info_load_from_filename( const char *filename, unsigned sm_version) +{ + std::string ptxas_filename(std::string(filename) + "as"); + char buff[1024], extra_flags[1024]; + extra_flags[0]=0; + if(!device_runtime->g_cdp_enabled) + snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version); + else + snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version); + snprintf(buff,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, filename, ptxas_filename.c_str()); + int result = system(buff); + if( result != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } - FILE *ptxinfo_in; - ptxinfo->g_ptxinfo_filename = strdup(ptxas_filename.c_str()); - ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename, "r"); - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); + FILE *ptxinfo_in; + ptxinfo->g_ptxinfo_filename = strdup(ptxas_filename.c_str()); + ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename,"r"); + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); } -void gpgpu_context::gpgpu_ptxinfo_load_from_string(const char *p_for_info, - unsigned source_num, - unsigned sm_version, - int no_of_ptx) { - // do ptxas for individual files instead of one big embedded ptx. This - // prevents the duplicate defs and declarations. - char ptx_file[1000]; - char *name = get_app_binary_name(); - char commandline[4096], fname[1024], fname2[1024], - final_tempfile_ptxinfo[1024], tempfile_ptxinfo[1024]; - for (int index = 1; index <= no_of_ptx; index++) { - snprintf(ptx_file, 1000, "%s.%d.sm_%u.ptx", name, index, sm_version); - snprintf(fname, 1024, "_ptx_XXXXXX"); - int fd = mkstemp(fname); - close(fd); +void gpgpu_context::gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num, unsigned sm_version, int no_of_ptx ) +{ + //do ptxas for individual files instead of one big embedded ptx. This prevents the duplicate defs and declarations. + char ptx_file[1000]; + char *name=get_app_binary_name(); + char commandline[4096], fname[1024], fname2[1024], final_tempfile_ptxinfo[1024], tempfile_ptxinfo[1024]; + for (int index=1; index <= no_of_ptx; index++){ + snprintf(ptx_file, 1000, "%s.%d.sm_%u.ptx", name, index, sm_version); + snprintf(fname,1024,"_ptx_XXXXXX"); + int fd=mkstemp(fname); + close(fd); - printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", - fname); - snprintf(commandline, 4096, "cat %s > %s", ptx_file, fname); - if (system(commandline) != 0) { - printf("ERROR: %s command failed\n", commandline); - exit(0); - } - - snprintf(fname2, 1024, "_ptx2_XXXXXX"); - fd = mkstemp(fname2); - close(fd); - char commandline2[4096]; - snprintf(commandline2, 4096, - "cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, " - "texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 " - "\\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed " - "'s/const\\[.\\]/const\\[0\\]/g' > %s", - fname, fname2); - printf("Running: %s\n", commandline2); - int result = system(commandline2); - if (result != 0) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); - printf( - " Ensure you have write access to simulation " - "directory\n"); - printf(" and have \'cat\' and \'sed\' in your path.\n"); - exit(1); - } + printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname); + snprintf(commandline,4096,"cat %s > %s",ptx_file, fname); + if (system(commandline) !=0) { + printf("ERROR: %s command failed\n", commandline); + exit(0); + } + + snprintf(fname2,1024,"_ptx2_XXXXXX"); + fd=mkstemp(fname2); + close(fd); + char commandline2[4096]; + snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2); + printf("Running: %s\n", commandline2); + int result = system(commandline2); + if( result != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); + printf(" Ensure you have write access to simulation directory\n"); + printf(" and have \'cat\' and \'sed\' in your path.\n"); + exit(1); + } - snprintf(tempfile_ptxinfo, 1024, "%sinfo", fname); - char extra_flags[1024]; - extra_flags[0] = 0; + snprintf(tempfile_ptxinfo,1024,"%sinfo",fname); + char extra_flags[1024]; + extra_flags[0]=0; #if CUDART_VERSION >= 3000 - if (g_occupancy_sm_number == 0) { - fprintf( - stderr, - "gpgpusim.config must specify the sm version for the GPU that you " - "use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n" - "The register file size is specifically tied to the sm version used " - "to querry ptxas for register usage.\n" - "A register size/SM mismatch may result in occupancy differences."); - exit(1); + if ( g_occupancy_sm_number == 0 ) { + fprintf( stderr, "gpgpusim.config must specify the sm version for the GPU that you use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n" + "The register file size is specifically tied to the sm version used to querry ptxas for register usage.\n" + "A register size/SM mismatch may result in occupancy differences." ); + exit(1); } - if (!device_runtime->g_cdp_enabled) - snprintf(extra_flags, 1024, "--gpu-name=sm_%u", g_occupancy_sm_number); + if(!device_runtime->g_cdp_enabled) + snprintf(extra_flags,1024,"--gpu-name=sm_%u", g_occupancy_sm_number); else - snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", - g_occupancy_sm_number); + snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",g_occupancy_sm_number); #endif - snprintf(commandline, 1024, - "$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file " - "/dev/null 2> %s", + snprintf(commandline,1024,"$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", extra_flags, fname2, tempfile_ptxinfo); printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); result = system(commandline); - if (result != 0) { - // 65280 = duplicate errors - if (result == 65280) { - FILE *ptxinfo_in; - ptxinfo_in = fopen(tempfile_ptxinfo, "r"); - ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); + if( result != 0 ) { + // 65280 = duplicate errors + if (result == 65280) { + FILE *ptxinfo_in; + ptxinfo_in = fopen(tempfile_ptxinfo,"r"); + ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); - fix_duplicate_errors(fname2); - snprintf(commandline, 1024, - "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file " - "/dev/null 2> %s", - extra_flags, fname2, tempfile_ptxinfo); - printf("GPGPU-Sim PTX: regenerating ptxinfo using \"%s\"\n", - commandline); - result = system(commandline); - } - if (result != 0) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); - } + fix_duplicate_errors(fname2); + snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: regenerating ptxinfo using \"%s\"\n", commandline); + result = system(commandline); + } + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } + } } - } - // TODO: duplicate code! move it into a function so that it can be reused! - if (no_of_ptx == 0) { - // For CDP, we dump everything. So no_of_ptx will be 0. - snprintf(fname, 1024, "_ptx_XXXXXX"); - int fd = mkstemp(fname); - close(fd); + //TODO: duplicate code! move it into a function so that it can be reused! + if(no_of_ptx==0) { + //For CDP, we dump everything. So no_of_ptx will be 0. + snprintf(fname,1024,"_ptx_XXXXXX"); + int fd=mkstemp(fname); + close(fd); - printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", - fname); - FILE *ptxfile = fopen(fname, "w"); - fprintf(ptxfile, "%s", p_for_info); - fclose(ptxfile); + printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname); + FILE *ptxfile = fopen(fname,"w"); + fprintf(ptxfile,"%s", p_for_info); + fclose(ptxfile); - snprintf(fname2, 1024, "_ptx2_XXXXXX"); - fd = mkstemp(fname2); - close(fd); - char commandline2[4096]; - snprintf(commandline2, 4096, - "cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, " - "texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 " - "\\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed " - "'s/const\\[.\\]/const\\[0\\]/g' > %s", - fname, fname2); - printf("Running: %s\n", commandline2); - int result = system(commandline2); - if (result != 0) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); - printf( - " Ensure you have write access to simulation " - "directory\n"); - printf(" and have \'cat\' and \'sed\' in your path.\n"); - exit(1); - } - // char tempfile_ptxinfo[1024]; - snprintf(tempfile_ptxinfo, 1024, "%sinfo", fname); - char extra_flags[1024]; - extra_flags[0] = 0; + snprintf(fname2,1024,"_ptx2_XXXXXX"); + fd=mkstemp(fname2); + close(fd); + char commandline2[4096]; + snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2); + printf("Running: %s\n", commandline2); + int result = system(commandline2); + if( result != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); + printf(" Ensure you have write access to simulation directory\n"); + printf(" and have \'cat\' and \'sed\' in your path.\n"); + exit(1); + } + //char tempfile_ptxinfo[1024]; + snprintf(tempfile_ptxinfo,1024,"%sinfo",fname); + char extra_flags[1024]; + extra_flags[0]=0; -#if CUDART_VERSION >= 3000 - if (sm_version == 0) sm_version = 20; - if (!device_runtime->g_cdp_enabled) - snprintf(extra_flags, 1024, "--gpu-name=sm_%u", sm_version); - else - snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", - sm_version); -#endif + #if CUDART_VERSION >= 3000 + if (sm_version == 0) sm_version = 20; + if(!device_runtime->g_cdp_enabled) + snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version); + else + snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version); + #endif - snprintf( - commandline, 1024, - "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", - extra_flags, fname2, tempfile_ptxinfo); - printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); - fflush(stdout); - result = system(commandline); - if (result != 0) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); + snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); + fflush(stdout); + result = system(commandline); + if( result != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } } - } - // Now that we got resource usage per kernel in a ptx file, we dump all into - // one file and pass it to rest of the code as usual. - if (no_of_ptx > 0) { - char commandline3[4096]; - snprintf(final_tempfile_ptxinfo, 1024, "f_tempfile_ptx"); - snprintf(commandline3, 4096, "cat *info > %s", final_tempfile_ptxinfo); - if (system(commandline3) != 0) { - printf("ERROR: Either we dont have info files or cat is not working \n"); - printf("ERROR: %s command failed\n", commandline3); - exit(1); - } - } + //Now that we got resource usage per kernel in a ptx file, we dump all into one file and pass it to rest of the code as usual. + if(no_of_ptx>0){ + char commandline3[4096]; + snprintf(final_tempfile_ptxinfo,1024,"f_tempfile_ptx"); + snprintf(commandline3,4096, "cat *info > %s", final_tempfile_ptxinfo); + if (system(commandline3)!=0) { + printf("ERROR: Either we dont have info files or cat is not working \n"); + printf("ERROR: %s command failed\n",commandline3); + exit(1); + } + } - if (no_of_ptx > 0) - ptxinfo->g_ptxinfo_filename = final_tempfile_ptxinfo; - else - ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; - FILE *ptxinfo_in; - ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename, "r"); + if(no_of_ptx>0) + ptxinfo->g_ptxinfo_filename = final_tempfile_ptxinfo; + else + ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; + FILE *ptxinfo_in; + ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename,"r"); - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); - snprintf(commandline, 1024, "rm -f *info"); - if (system(commandline) != 0) { - printf("GPGPU-Sim PTX: ERROR ** while removing temporary info files\n"); - exit(1); - } - if (!g_save_embedded_ptx) { - if (no_of_ptx > 0) - snprintf(commandline, 1024, "rm -f %s %s %s", fname, fname2, - final_tempfile_ptxinfo); - else - snprintf(commandline, 1024, "rm -f %s %s %s", fname, fname2, - tempfile_ptxinfo); - printf("GPGPU-Sim PTX: removing ptxinfo using \"%s\"\n", commandline); - if (system(commandline) != 0) { - printf("GPGPU-Sim PTX: ERROR ** while removing temporary files\n"); - exit(1); + snprintf(commandline,1024,"rm -f *info"); + if( system(commandline) != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while removing temporary info files\n"); + exit(1); + } + if( ! g_save_embedded_ptx ) { + if(no_of_ptx>0) + snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, final_tempfile_ptxinfo); + else + snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: removing ptxinfo using \"%s\"\n", commandline); + if( system(commandline) != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while removing temporary files\n"); + exit(1); + } } - } } diff --git a/src/cuda-sim/ptx_loader.h b/src/cuda-sim/ptx_loader.h index 17c4524..d8f1cbc 100644 --- a/src/cuda-sim/ptx_loader.h +++ b/src/cuda-sim/ptx_loader.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -33,22 +31,22 @@ #define PTXINFO_LINEBUF_SIZE 1024 class gpgpu_context; -typedef void* yyscan_t; -class ptxinfo_data { - public: - ptxinfo_data(gpgpu_context* ctx) { gpgpu_ctx = ctx; } - yyscan_t scanner; - char linebuf[PTXINFO_LINEBUF_SIZE]; - unsigned col; - const char* g_ptxinfo_filename; - class gpgpu_context* gpgpu_ctx; - bool g_keep_intermediate_files; - bool m_ptx_save_converted_ptxplus; - void ptxinfo_addinfo(); - bool keep_intermediate_files(); - char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus( - const std::string ptx_str, const std::string sass_str, - const std::string elf_str); +typedef void * yyscan_t; +class ptxinfo_data{ + public: + ptxinfo_data(gpgpu_context* ctx) { + gpgpu_ctx = ctx; + } + yyscan_t scanner; + char linebuf[PTXINFO_LINEBUF_SIZE]; + unsigned col; + const char *g_ptxinfo_filename; + class gpgpu_context* gpgpu_ctx; + bool g_keep_intermediate_files; + bool m_ptx_save_converted_ptxplus; + void ptxinfo_addinfo(); + bool keep_intermediate_files(); + char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptx_str, const std::string sass_str, const std::string elf_str); }; #endif diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index c135a6c..a4f4a0c 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -28,971 +26,998 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "ptx_parser.h" -#include "../../libcuda/gpgpu_context.h" #include "ptx_ir.h" +#include "../../libcuda/gpgpu_context.h" -typedef void *yyscan_t; -#include <stdarg.h> +typedef void * yyscan_t; #include "ptx.tab.h" +#include <stdarg.h> -extern int ptx_get_lineno(yyscan_t yyscanner); -extern YYSTYPE *ptx_get_lval(yyscan_t yyscanner); -extern int ptx_error(yyscan_t yyscanner, const char *s); -extern int ptx_lex_init(yyscan_t *scanner); -extern void ptx_set_in(FILE *_in_str, yyscan_t yyscanner); -extern FILE *ptx_get_in(yyscan_t yyscanner); -extern int ptx_parse(yyscan_t scanner, ptx_recognizer *recognizer); +extern int ptx_get_lineno (yyscan_t yyscanner ); +extern YYSTYPE* ptx_get_lval (yyscan_t yyscanner ); +extern int ptx_error( yyscan_t yyscanner, const char *s ); +extern int ptx_lex_init(yyscan_t* scanner); +extern void ptx_set_in(FILE * _in_str ,yyscan_t yyscanner ); +extern FILE *ptx_get_in (yyscan_t yyscanner ); +extern int ptx_parse(yyscan_t scanner, ptx_recognizer* recognizer); extern int ptx_lex_destroy(yyscan_t scanner); -void ptx_recognizer::set_ptx_warp_size(const struct core_config *warp_size) { - g_shader_core_config = warp_size; +void ptx_recognizer::set_ptx_warp_size(const struct core_config * warp_size) +{ + g_shader_core_config=warp_size; } -#define PTX_PARSE_DPRINTF(...) \ - if (g_debug_ir_generation) { \ - printf(" %s:%u => ", gpgpu_ctx->g_filename, ptx_get_lineno(scanner)); \ - printf(" (%s:%u) ", __FILE__, __LINE__); \ - printf(__VA_ARGS__); \ - printf("\n"); \ - fflush(stdout); \ - } -static std::map<unsigned, std::string> g_ptx_token_decode; +#define PTX_PARSE_DPRINTF(...) \ + if( g_debug_ir_generation ) { \ + printf(" %s:%u => ",gpgpu_ctx->g_filename,ptx_get_lineno(scanner)); \ + printf(" (%s:%u) ", __FILE__, __LINE__); \ + printf(__VA_ARGS__); \ + printf("\n"); \ + fflush(stdout); \ + } + +static std::map<unsigned,std::string> g_ptx_token_decode; -const char *decode_token(int type) { return g_ptx_token_decode[type].c_str(); } +const char *decode_token( int type ) +{ + return g_ptx_token_decode[type].c_str(); +} -void ptx_recognizer::read_parser_environment_variables() { - gpgpu_ctx->g_filename = getenv("PTX_SIM_KERNELFILE"); - char *dbg_level = getenv("PTX_SIM_DEBUG"); - if (dbg_level && strlen(dbg_level)) { - int debug_execution = 0; - sscanf(dbg_level, "%d", &debug_execution); - if (debug_execution >= 30) g_debug_ir_generation = true; - } +void ptx_recognizer::read_parser_environment_variables() +{ + gpgpu_ctx->g_filename = getenv("PTX_SIM_KERNELFILE"); + char *dbg_level = getenv("PTX_SIM_DEBUG"); + if ( dbg_level && strlen(dbg_level) ) { + int debug_execution=0; + sscanf(dbg_level,"%d", &debug_execution); + if ( debug_execution >= 30 ) + g_debug_ir_generation=true; + } } -void ptx_recognizer::init_directive_state() { - PTX_PARSE_DPRINTF("init_directive_state"); - g_space_spec = undefined_space; - g_ptr_spec = undefined_space; - g_scalar_type_spec = -1; - g_vector_spec = -1; - g_opcode = -1; - g_alignment_spec = -1; - g_size = -1; - g_extern_spec = 0; - g_scalar_type.clear(); - g_operands.clear(); - g_last_symbol = NULL; +void ptx_recognizer::init_directive_state() +{ + PTX_PARSE_DPRINTF("init_directive_state"); + g_space_spec=undefined_space; + g_ptr_spec=undefined_space; + g_scalar_type_spec=-1; + g_vector_spec=-1; + g_opcode=-1; + g_alignment_spec = -1; + g_size = -1; + g_extern_spec = 0; + g_scalar_type.clear(); + g_operands.clear(); + g_last_symbol = NULL; } -void ptx_recognizer::init_instruction_state() { - PTX_PARSE_DPRINTF("init_instruction_state"); - g_pred = NULL; - g_neg_pred = 0; - g_pred_mod = -1; - g_label = NULL; - g_opcode = -1; - g_options.clear(); - g_wmma_options.clear(); - g_return_var = operand_info(gpgpu_ctx); - init_directive_state(); +void ptx_recognizer::init_instruction_state() +{ + PTX_PARSE_DPRINTF("init_instruction_state"); + g_pred = NULL; + g_neg_pred = 0; + g_pred_mod = -1; + g_label = NULL; + g_opcode = -1; + g_options.clear(); + g_wmma_options.clear(); + g_return_var = operand_info(gpgpu_ctx); + init_directive_state(); } -symbol_table *gpgpu_context::init_parser(const char *ptx_filename) { - g_filename = strdup(ptx_filename); - if (g_global_allfiles_symbol_table == NULL) { - g_global_allfiles_symbol_table = - new symbol_table("global_allfiles", 0, NULL, this); - ptx_parser->g_global_symbol_table = ptx_parser->g_current_symbol_table = - g_global_allfiles_symbol_table; - } -/*else { - g_global_symbol_table = g_current_symbol_table = new -symbol_table("global",0,g_global_allfiles_symbol_table); -}*/ +symbol_table * gpgpu_context::init_parser( const char *ptx_filename ) +{ + g_filename = strdup(ptx_filename); + if (g_global_allfiles_symbol_table == NULL) { + g_global_allfiles_symbol_table = new symbol_table("global_allfiles", 0, NULL, this); + ptx_parser->g_global_symbol_table = ptx_parser->g_current_symbol_table = g_global_allfiles_symbol_table; + } + /*else { + g_global_symbol_table = g_current_symbol_table = new symbol_table("global",0,g_global_allfiles_symbol_table); + }*/ -#define DEF(X, Y) g_ptx_token_decode[X] = Y; +#define DEF(X,Y) g_ptx_token_decode[X] = Y; #include "ptx_parser_decode.def" #undef DEF - g_ptx_token_decode[undefined_space] = "undefined_space"; - g_ptx_token_decode[undefined_space] = "undefined_space=0"; - g_ptx_token_decode[reg_space] = "reg_space"; - g_ptx_token_decode[local_space] = "local_space"; - g_ptx_token_decode[shared_space] = "shared_space"; - g_ptx_token_decode[param_space_unclassified] = "param_space_unclassified"; - g_ptx_token_decode[param_space_kernel] = "param_space_kernel"; - g_ptx_token_decode[param_space_local] = "param_space_local"; - g_ptx_token_decode[const_space] = "const_space"; - g_ptx_token_decode[tex_space] = "tex_space"; - g_ptx_token_decode[surf_space] = "surf_space"; - g_ptx_token_decode[global_space] = "global_space"; - g_ptx_token_decode[generic_space] = "generic_space"; - g_ptx_token_decode[instruction_space] = "instruction_space"; + g_ptx_token_decode[undefined_space] = "undefined_space"; + g_ptx_token_decode[undefined_space] = "undefined_space=0"; + g_ptx_token_decode[reg_space] = "reg_space"; + g_ptx_token_decode[local_space] = "local_space"; + g_ptx_token_decode[shared_space] = "shared_space"; + g_ptx_token_decode[param_space_unclassified] = "param_space_unclassified"; + g_ptx_token_decode[param_space_kernel] = "param_space_kernel"; + g_ptx_token_decode[param_space_local] = "param_space_local"; + g_ptx_token_decode[const_space] = "const_space"; + g_ptx_token_decode[tex_space] = "tex_space"; + g_ptx_token_decode[surf_space] = "surf_space"; + g_ptx_token_decode[global_space] = "global_space"; + g_ptx_token_decode[generic_space] = "generic_space"; + g_ptx_token_decode[instruction_space] = "instruction_space"; - ptx_lex_init(&(ptx_parser->scanner)); - ptx_parser->init_directive_state(); - ptx_parser->init_instruction_state(); + ptx_lex_init(&(ptx_parser->scanner)); + ptx_parser->init_directive_state(); + ptx_parser->init_instruction_state(); - FILE *ptx_in; - ptx_in = fopen(ptx_filename, "r"); - ptx_set_in(ptx_in, ptx_parser->scanner); - ptx_parse(ptx_parser->scanner, ptx_parser); - ptx_in = ptx_get_in(ptx_parser->scanner); - ptx_lex_destroy(ptx_parser->scanner); - fclose(ptx_in); - return ptx_parser->g_global_symbol_table; + FILE *ptx_in; + ptx_in = fopen(ptx_filename, "r"); + ptx_set_in(ptx_in, ptx_parser->scanner); + ptx_parse(ptx_parser->scanner, ptx_parser); + ptx_in = ptx_get_in(ptx_parser->scanner); + ptx_lex_destroy(ptx_parser->scanner); + fclose(ptx_in); + return ptx_parser->g_global_symbol_table; } -void ptx_recognizer::start_function(int entry_point) { - PTX_PARSE_DPRINTF("start_function"); - init_directive_state(); - init_instruction_state(); - g_entry_point = entry_point; - g_func_info = NULL; - g_entry_func_param_index = 0; + +void ptx_recognizer::start_function( int entry_point ) +{ + PTX_PARSE_DPRINTF("start_function"); + init_directive_state(); + init_instruction_state(); + g_entry_point = entry_point; + g_func_info = NULL; + g_entry_func_param_index=0; } -void ptx_recognizer::add_function_name(const char *name) { - PTX_PARSE_DPRINTF( - "add_function_name %s %s", name, - ((g_entry_point == 1) ? "(entrypoint)" - : ((g_entry_point == 2) ? "(extern)" : ""))); - bool prior_decl = g_global_symbol_table->add_function_decl( - name, g_entry_point, &g_func_info, &g_current_symbol_table); - if (g_add_identifier_cached__identifier) { - add_identifier(g_add_identifier_cached__identifier, - g_add_identifier_cached__array_dim, - g_add_identifier_cached__array_ident); - free(g_add_identifier_cached__identifier); - g_add_identifier_cached__identifier = NULL; - g_func_info->add_return_var(g_last_symbol); - init_directive_state(); - } - if (prior_decl) { - g_func_info->remove_args(); - } - g_global_symbol_table->add_function(g_func_info, gpgpu_ctx->g_filename, - ptx_get_lineno(scanner)); +void ptx_recognizer::add_function_name( const char *name ) +{ + PTX_PARSE_DPRINTF("add_function_name %s %s", name, ((g_entry_point==1)?"(entrypoint)":((g_entry_point==2)?"(extern)":""))); + bool prior_decl = g_global_symbol_table->add_function_decl( name, g_entry_point, &g_func_info, &g_current_symbol_table ); + if( g_add_identifier_cached__identifier ) { + add_identifier( g_add_identifier_cached__identifier, + g_add_identifier_cached__array_dim, + g_add_identifier_cached__array_ident ); + free( g_add_identifier_cached__identifier ); + g_add_identifier_cached__identifier = NULL; + g_func_info->add_return_var( g_last_symbol ); + init_directive_state(); + } + if( prior_decl ) { + g_func_info->remove_args(); + } + g_global_symbol_table->add_function( g_func_info, gpgpu_ctx->g_filename, ptx_get_lineno(scanner) ); } -// Jin: handle instruction group for cdp +//Jin: handle instruction group for cdp void ptx_recognizer::start_inst_group() { - PTX_PARSE_DPRINTF("start_instruction_group"); - g_current_symbol_table = g_current_symbol_table->start_inst_group(); + PTX_PARSE_DPRINTF("start_instruction_group"); + g_current_symbol_table = g_current_symbol_table->start_inst_group(); } void ptx_recognizer::end_inst_group() { - PTX_PARSE_DPRINTF("end_instruction_group"); - g_current_symbol_table = g_current_symbol_table->end_inst_group(); + PTX_PARSE_DPRINTF("end_instruction_group"); + g_current_symbol_table = g_current_symbol_table->end_inst_group(); } -void ptx_recognizer::add_directive() { - PTX_PARSE_DPRINTF("add_directive"); - init_directive_state(); +void ptx_recognizer::add_directive() +{ + PTX_PARSE_DPRINTF("add_directive"); + init_directive_state(); } -#define mymax(a, b) ((a) > (b) ? (a) : (b)) +#define mymax(a,b) ((a)>(b)?(a):(b)) -void ptx_recognizer::end_function() { - PTX_PARSE_DPRINTF("end_function"); +void ptx_recognizer::end_function() +{ + PTX_PARSE_DPRINTF("end_function"); - init_directive_state(); - init_instruction_state(); - g_max_regs_per_thread = mymax(g_max_regs_per_thread, - (g_current_symbol_table->next_reg_num() - 1)); - g_func_info->add_inst(g_instructions); - g_instructions.clear(); - gpgpu_ptx_assemble(g_func_info->get_name(), g_func_info); - g_current_symbol_table = g_global_symbol_table; + init_directive_state(); + init_instruction_state(); + g_max_regs_per_thread = mymax( g_max_regs_per_thread, (g_current_symbol_table->next_reg_num()-1)); + g_func_info->add_inst( g_instructions ); + g_instructions.clear(); + gpgpu_ptx_assemble( g_func_info->get_name(), g_func_info ); + g_current_symbol_table = g_global_symbol_table; - PTX_PARSE_DPRINTF("function %s, PC = %d\n", g_func_info->get_name().c_str(), - g_func_info->get_start_PC()); + PTX_PARSE_DPRINTF("function %s, PC = %d\n", g_func_info->get_name().c_str(), g_func_info->get_start_PC()); } -#define parse_error(msg, ...) \ - parse_error_impl(__FILE__, __LINE__, msg, ##__VA_ARGS__) -#define parse_assert(cond, msg, ...) \ - parse_assert_impl((cond), __FILE__, __LINE__, msg, ##__VA_ARGS__) +#define parse_error(msg, ...) parse_error_impl(__FILE__,__LINE__, msg, ##__VA_ARGS__) +#define parse_assert(cond,msg, ...) parse_assert_impl((cond),__FILE__,__LINE__, msg, ##__VA_ARGS__) -void ptx_recognizer::parse_error_impl(const char *file, unsigned line, - const char *msg, ...) { - va_list ap; - char buf[1024]; - va_start(ap, msg); - vsnprintf(buf, 1024, msg, ap); - va_end(ap); +void ptx_recognizer::parse_error_impl( const char *file, unsigned line, const char *msg, ... ) +{ + va_list ap; + char buf[1024]; + va_start(ap,msg); + vsnprintf(buf,1024,msg,ap); + va_end(ap); - g_error_detected = 1; - printf("%s:%u: Parse error: %s (%s:%u)\n\n", gpgpu_ctx->g_filename, - ptx_get_lineno(scanner), buf, file, line); - ptx_error(scanner, NULL); - abort(); - exit(1); + g_error_detected = 1; + printf("%s:%u: Parse error: %s (%s:%u)\n\n", gpgpu_ctx->g_filename, ptx_get_lineno(scanner), buf, file, line); + ptx_error(scanner, NULL); + abort(); + exit(1); } -void ptx_recognizer::parse_assert_impl(int test_value, const char *file, - unsigned line, const char *msg, ...) { - va_list ap; - char buf[1024]; - va_start(ap, msg); - vsnprintf(buf, 1024, msg, ap); - va_end(ap); +void ptx_recognizer::parse_assert_impl( int test_value, const char *file, unsigned line, const char *msg, ... ) +{ + va_list ap; + char buf[1024]; + va_start(ap,msg); + vsnprintf(buf,1024,msg,ap); + va_end(ap); - if (test_value == 0) parse_error_impl(file, line, msg); + if ( test_value == 0 ) + parse_error_impl(file,line, msg); } -void ptx_recognizer::set_return() { - parse_assert((g_opcode == CALL_OP || g_opcode == CALLP_OP), - "only call can have return value"); - g_operands.front().set_return(); - g_return_var = g_operands.front(); + + +void ptx_recognizer::set_return() +{ + parse_assert( (g_opcode == CALL_OP || g_opcode == CALLP_OP), "only call can have return value"); + g_operands.front().set_return(); + g_return_var = g_operands.front(); } -const ptx_instruction *ptx_recognizer::ptx_instruction_lookup( - const char *filename, unsigned linenumber) { - std::map<std::string, std::map<unsigned, const ptx_instruction *> >::iterator - f = g_inst_lookup.find(filename); - if (f == g_inst_lookup.end()) return NULL; - std::map<unsigned, const ptx_instruction *>::iterator l = - f->second.find(linenumber); - if (l == f->second.end()) return NULL; - return l->second; + +const ptx_instruction *ptx_recognizer::ptx_instruction_lookup( const char *filename, unsigned linenumber ) +{ + std::map<std::string,std::map<unsigned,const ptx_instruction*> >::iterator f=g_inst_lookup.find(filename); + if( f == g_inst_lookup.end() ) + return NULL; + std::map<unsigned,const ptx_instruction*>::iterator l=f->second.find(linenumber); + if( l == f->second.end() ) + return NULL; + return l->second; } -void ptx_recognizer::add_instruction() { - PTX_PARSE_DPRINTF("add_instruction: %s", - ((g_opcode > 0) ? g_opcode_string[g_opcode] : "<label>")); - assert(g_shader_core_config != 0); - ptx_instruction *i = new ptx_instruction( - g_opcode, g_pred, g_neg_pred, g_pred_mod, g_label, g_operands, - g_return_var, g_options, g_wmma_options, g_scalar_type, g_space_spec, - gpgpu_ctx->g_filename, ptx_get_lineno(scanner), linebuf, - g_shader_core_config, gpgpu_ctx); - g_instructions.push_back(i); - g_inst_lookup[gpgpu_ctx->g_filename][ptx_get_lineno(scanner)] = i; - init_instruction_state(); +void ptx_recognizer::add_instruction() +{ + PTX_PARSE_DPRINTF("add_instruction: %s", ((g_opcode>0)?g_opcode_string[g_opcode]:"<label>") ); + assert( g_shader_core_config != 0 ); + ptx_instruction *i = new ptx_instruction( g_opcode, + g_pred, + g_neg_pred, + g_pred_mod, + g_label, + g_operands, + g_return_var, + g_options, + g_wmma_options, + g_scalar_type, + g_space_spec, + gpgpu_ctx->g_filename, + ptx_get_lineno(scanner), + linebuf, + g_shader_core_config, + gpgpu_ctx ); + g_instructions.push_back(i); + g_inst_lookup[gpgpu_ctx->g_filename][ptx_get_lineno(scanner)] = i; + init_instruction_state(); } -void ptx_recognizer::add_variables() { - PTX_PARSE_DPRINTF("add_variables"); - if (!g_operands.empty()) { - assert(g_last_symbol != NULL); - g_last_symbol->add_initializer(g_operands); - } - init_directive_state(); +void ptx_recognizer::add_variables() +{ + PTX_PARSE_DPRINTF("add_variables"); + if ( !g_operands.empty() ) { + assert( g_last_symbol != NULL ); + g_last_symbol->add_initializer(g_operands); + } + init_directive_state(); } -void ptx_recognizer::set_variable_type() { - PTX_PARSE_DPRINTF("set_variable_type space_spec=%s scalar_type_spec=%s", - g_ptx_token_decode[g_space_spec.get_type()].c_str(), - g_ptx_token_decode[g_scalar_type_spec].c_str()); - parse_assert(g_space_spec != undefined_space, - "variable has no space specification"); - parse_assert( - g_scalar_type_spec != -1, - "variable has no type information"); // need to extend for structs? - g_var_type = g_current_symbol_table->add_type( - g_space_spec, g_scalar_type_spec, g_vector_spec, g_alignment_spec, - g_extern_spec); +void ptx_recognizer::set_variable_type() +{ + PTX_PARSE_DPRINTF("set_variable_type space_spec=%s scalar_type_spec=%s", + g_ptx_token_decode[g_space_spec.get_type()].c_str(), + g_ptx_token_decode[g_scalar_type_spec].c_str() ); + parse_assert( g_space_spec != undefined_space, "variable has no space specification" ); + parse_assert( g_scalar_type_spec != -1, "variable has no type information" ); // need to extend for structs? + g_var_type = g_current_symbol_table->add_type( g_space_spec, + g_scalar_type_spec, + g_vector_spec, + g_alignment_spec, + g_extern_spec ); } -bool ptx_recognizer::check_for_duplicates(const char *identifier) { - const symbol *s = g_current_symbol_table->lookup(identifier); - return (s != NULL); +bool ptx_recognizer::check_for_duplicates( const char *identifier ) +{ + const symbol *s = g_current_symbol_table->lookup(identifier); + return ( s != NULL ); } -// Returns padding that needs to be inserted ahead of address to make it aligned -// to min(size, maxalign) +// Returns padding that needs to be inserted ahead of address to make it aligned to min(size, maxalign) /* * @param address the address in bytes * @param size the size of the memory to be allocated in bytes - * @param maximum alignment in bytes. i.e. if size is too big then align to this - * instead + * @param maximum alignment in bytes. i.e. if size is too big then align to this instead */ -int pad_address(new_addr_type address, unsigned size, unsigned maxalign) { - assert(size >= 0); - assert(maxalign > 0); - int alignto = maxalign; - if (size < maxalign && (size & (size - 1)) == 0) { // size is a power of 2 - alignto = size; - } - return alignto ? ((alignto - (address % alignto)) % alignto) : 0; +int pad_address (new_addr_type address, unsigned size, unsigned maxalign) { + assert(size >= 0); + assert(maxalign > 0); + int alignto = maxalign; + if (size < maxalign && + (size & (size-1)) == 0) { //size is a power of 2 + alignto = size; + } + return alignto ? ((alignto - (address % alignto)) % alignto) : 0; } -void ptx_recognizer::add_identifier(const char *identifier, int array_dim, - unsigned array_ident) { - if (array_ident == ARRAY_IDENTIFIER) { - g_size *= array_dim; - } - if (g_func_decl && (g_func_info == NULL)) { - // return variable decl... - assert(g_add_identifier_cached__identifier == NULL); - g_add_identifier_cached__identifier = strdup(identifier); - g_add_identifier_cached__array_dim = array_dim; - g_add_identifier_cached__array_ident = array_ident; - return; - } - PTX_PARSE_DPRINTF("add_identifier \"%s\" (%u)", identifier, g_ident_add_uid); - g_ident_add_uid++; - type_info *type = g_var_type; - type_info_key ti = type->get_key(); - int basic_type; - int regnum; - size_t num_bits; - unsigned addr_pad; - new_addr_type addr; - ti.type_decode(num_bits, basic_type); +void ptx_recognizer::add_identifier( const char *identifier, int array_dim, unsigned array_ident ) +{ + if(array_ident==ARRAY_IDENTIFIER){ + g_size *= array_dim; + } + if( g_func_decl && (g_func_info == NULL) ) { + // return variable decl... + assert( g_add_identifier_cached__identifier == NULL ); + g_add_identifier_cached__identifier = strdup(identifier); + g_add_identifier_cached__array_dim = array_dim; + g_add_identifier_cached__array_ident = array_ident; + return; + } + PTX_PARSE_DPRINTF("add_identifier \"%s\" (%u)", identifier, g_ident_add_uid); + g_ident_add_uid++; + type_info *type = g_var_type; + type_info_key ti = type->get_key(); + int basic_type; + int regnum; + size_t num_bits; + unsigned addr_pad; + new_addr_type addr; + ti.type_decode(num_bits,basic_type); - bool duplicates = check_for_duplicates(identifier); - if (duplicates) { - symbol *s = g_current_symbol_table->lookup(identifier); - g_last_symbol = s; - if (g_func_decl) return; - std::string msg = std::string(identifier) + " was declared previous at " + - s->decl_location() + " skipping new declaration"; - printf("GPGPU-Sim PTX: Warning %s\n", msg.c_str()); - return; - } + bool duplicates = check_for_duplicates( identifier ); + if( duplicates ) { + symbol *s = g_current_symbol_table->lookup(identifier); + g_last_symbol = s; + if( g_func_decl ) + return; + std::string msg = std::string(identifier) + " was declared previous at " + s->decl_location() + " skipping new declaration"; + printf("GPGPU-Sim PTX: Warning %s\n", msg.c_str()); + return; + } - assert(g_var_type != NULL); - switch (array_ident) { - case ARRAY_IDENTIFIER: - type = g_current_symbol_table->get_array_type(type, array_dim); + assert( g_var_type != NULL ); + switch ( array_ident ) { + case ARRAY_IDENTIFIER: + type = g_current_symbol_table->get_array_type(type,array_dim); num_bits = array_dim * num_bits; break; - case ARRAY_IDENTIFIER_NO_DIM: - type = g_current_symbol_table->get_array_type(type, (unsigned)-1); + case ARRAY_IDENTIFIER_NO_DIM: + type = g_current_symbol_table->get_array_type(type,(unsigned)-1); num_bits = 0; break; - default: + default: break; - } - g_last_symbol = g_current_symbol_table->add_variable( - identifier, type, num_bits / 8, gpgpu_ctx->g_filename, - ptx_get_lineno(scanner)); - switch (ti.get_memory_space().get_type()) { - case reg_space: { + } + g_last_symbol = g_current_symbol_table->add_variable(identifier,type,num_bits/8,gpgpu_ctx->g_filename,ptx_get_lineno(scanner)); + switch ( ti.get_memory_space().get_type() ) { + case reg_space: { regnum = g_current_symbol_table->next_reg_num(); int arch_regnum = -1; for (int d = 0; d < strlen(identifier); d++) { - if (isdigit(identifier[d])) { - sscanf(identifier + d, "%d", &arch_regnum); - break; - } + if (isdigit(identifier[d])) { + sscanf(identifier + d, "%d", &arch_regnum); + break; + } } if (strcmp(identifier, "%sp") == 0) { - arch_regnum = 0; + arch_regnum = 0; } g_last_symbol->set_regno(regnum, arch_regnum); - } break; - case shared_space: - printf("GPGPU-Sim PTX: allocating shared region for \"%s\" ", identifier); + } break; + case shared_space: + printf("GPGPU-Sim PTX: allocating shared region for \"%s\" ", + identifier); fflush(stdout); - assert((num_bits % 8) == 0); + assert( (num_bits%8) == 0 ); addr = g_current_symbol_table->get_shared_next(); - addr_pad = pad_address(addr, num_bits / 8, 128); - printf("from 0x%llx to 0x%llx (shared memory space)\n", addr + addr_pad, - addr + addr_pad + num_bits / 8); - fflush(stdout); - g_last_symbol->set_address(addr + addr_pad); - g_current_symbol_table->alloc_shared(num_bits / 8 + addr_pad); - break; - case sstarr_space: - printf("GPGPU-Sim PTX: allocating sstarr region for \"%s\" ", identifier); - fflush(stdout); - assert((num_bits % 8) == 0); - addr = g_current_symbol_table->get_sstarr_next(); - addr_pad = pad_address(addr, num_bits / 8, 128); - printf("from 0x%llx to 0x%llx (sstarr memory space)\n", addr + addr_pad, - addr + addr_pad + num_bits / 8); - fflush(stdout); - g_last_symbol->set_address(addr + addr_pad); - g_current_symbol_table->alloc_sstarr(num_bits / 8 + addr_pad); + addr_pad = pad_address(addr, num_bits/8, 128); + printf("from 0x%llx to 0x%llx (shared memory space)\n", + addr+addr_pad, + addr+addr_pad + num_bits/8); + fflush(stdout); + g_last_symbol->set_address( addr+addr_pad ); + g_current_symbol_table->alloc_shared( num_bits/8 + addr_pad ); break; - case const_space: - if (array_ident == ARRAY_IDENTIFIER_NO_DIM) { - printf( - "GPGPU-Sim PTX: deferring allocation of constant region for \"%s\" " - "(need size information)\n", - identifier); + case sstarr_space: + printf("GPGPU-Sim PTX: allocating sstarr region for \"%s\" ", + identifier); + fflush(stdout); + assert( (num_bits%8) == 0 ); + addr = g_current_symbol_table->get_sstarr_next(); + addr_pad = pad_address(addr, num_bits/8, 128); + printf("from 0x%llx to 0x%llx (sstarr memory space)\n", + addr+addr_pad, + addr+addr_pad + num_bits/8); + fflush(stdout); + g_last_symbol->set_address( addr+addr_pad ); + g_current_symbol_table->alloc_sstarr( num_bits/8 + addr_pad ); + break; + case const_space: + if( array_ident == ARRAY_IDENTIFIER_NO_DIM ) { + printf("GPGPU-Sim PTX: deferring allocation of constant region for \"%s\" (need size information)\n", identifier ); } else { - printf("GPGPU-Sim PTX: allocating constant region for \"%s\" ", - identifier); - fflush(stdout); - assert((num_bits % 8) == 0); - addr = g_current_symbol_table->get_global_next(); - addr_pad = pad_address(addr, num_bits / 8, 128); - printf("from 0x%llx to 0x%llx (global memory space) %u\n", - addr + addr_pad, addr + addr_pad + num_bits / 8, - g_const_alloc++); - fflush(stdout); - g_last_symbol->set_address(addr + addr_pad); - g_current_symbol_table->alloc_global(num_bits / 8 + addr_pad); + printf("GPGPU-Sim PTX: allocating constant region for \"%s\" ", + identifier); + fflush(stdout); + assert( (num_bits%8) == 0 ); + addr = g_current_symbol_table->get_global_next(); + addr_pad = pad_address(addr, num_bits/8, 128); + printf("from 0x%llx to 0x%llx (global memory space) %u\n", + addr+addr_pad, + addr+addr_pad + num_bits/8, + g_const_alloc++); + fflush(stdout); + g_last_symbol->set_address( addr + addr_pad ); + g_current_symbol_table->alloc_global( num_bits/8 + addr_pad ); } - if (g_current_symbol_table == g_global_symbol_table) { - gpgpu_ctx->func_sim->g_constants.insert(identifier); + if( g_current_symbol_table == g_global_symbol_table ) { + gpgpu_ctx->func_sim->g_constants.insert( identifier ); } - assert(g_current_symbol_table != NULL); - g_sym_name_to_symbol_table[identifier] = g_current_symbol_table; + assert( g_current_symbol_table != NULL ); + g_sym_name_to_symbol_table[ identifier ] = g_current_symbol_table; break; - case global_space: - printf("GPGPU-Sim PTX: allocating global region for \"%s\" ", identifier); + case global_space: + printf("GPGPU-Sim PTX: allocating global region for \"%s\" ", + identifier); fflush(stdout); - assert((num_bits % 8) == 0); + assert( (num_bits%8) == 0 ); addr = g_current_symbol_table->get_global_next(); - addr_pad = pad_address(addr, num_bits / 8, 128); - printf("from 0x%llx to 0x%llx (global memory space)\n", addr + addr_pad, - addr + addr_pad + num_bits / 8); + addr_pad = pad_address(addr, num_bits/8, 128); + printf("from 0x%llx to 0x%llx (global memory space)\n", + addr+addr_pad, + addr+addr_pad + num_bits/8); fflush(stdout); - g_last_symbol->set_address(addr + addr_pad); - g_current_symbol_table->alloc_global(num_bits / 8 + addr_pad); - gpgpu_ctx->func_sim->g_globals.insert(identifier); - assert(g_current_symbol_table != NULL); - g_sym_name_to_symbol_table[identifier] = g_current_symbol_table; + g_last_symbol->set_address( addr+addr_pad ); + g_current_symbol_table->alloc_global( num_bits/8 + addr_pad ); + gpgpu_ctx->func_sim->g_globals.insert( identifier ); + assert( g_current_symbol_table != NULL ); + g_sym_name_to_symbol_table[ identifier ] = g_current_symbol_table; break; - case local_space: - if (g_func_info == NULL) { - printf("GPGPU-Sim PTX: allocating local region for \"%s\" ", - identifier); - fflush(stdout); - assert((num_bits % 8) == 0); - addr = g_current_symbol_table->get_local_next(); - addr_pad = pad_address(addr, num_bits / 8, 128); - printf("from 0x%llx to 0x%llx (local memory space)\n", addr + addr_pad, - addr + addr_pad + num_bits / 8); - fflush(stdout); - g_last_symbol->set_address(addr + addr_pad); - g_current_symbol_table->alloc_local(num_bits / 8 + addr_pad); + case local_space: + if( g_func_info == NULL ) { + printf("GPGPU-Sim PTX: allocating local region for \"%s\" ", identifier); + fflush(stdout); + assert( (num_bits%8) == 0 ); + addr = g_current_symbol_table->get_local_next(); + addr_pad = pad_address(addr, num_bits/8, 128); + printf("from 0x%llx to 0x%llx (local memory space)\n", + addr+addr_pad, + addr+addr_pad + num_bits/8); + fflush(stdout); + g_last_symbol->set_address( addr+addr_pad); + g_current_symbol_table->alloc_local( num_bits/8 + addr_pad); } else { - printf( - "GPGPU-Sim PTX: allocating stack frame region for .local \"%s\" ", - identifier); + printf("GPGPU-Sim PTX: allocating stack frame region for .local \"%s\" ", + identifier); fflush(stdout); - assert((num_bits % 8) == 0); + assert( (num_bits%8) == 0 ); addr = g_current_symbol_table->get_local_next(); - addr_pad = pad_address(addr, num_bits / 8, 128); - printf("from 0x%llx to 0x%llx\n", addr + addr_pad, - addr + addr_pad + num_bits / 8); + addr_pad = pad_address(addr, num_bits/8, 128); + printf("from 0x%llx to 0x%llx\n", + addr+addr_pad, + addr+addr_pad + num_bits/8); fflush(stdout); - g_last_symbol->set_address(addr + addr_pad); - g_current_symbol_table->alloc_local(num_bits / 8 + addr_pad); - g_func_info->set_framesize(g_current_symbol_table->get_local_next()); + g_last_symbol->set_address( addr+addr_pad ); + g_current_symbol_table->alloc_local( num_bits/8 + addr_pad); + g_func_info->set_framesize( g_current_symbol_table->get_local_next() ); } break; - case tex_space: + case tex_space: printf("GPGPU-Sim PTX: encountered texture directive %s.\n", identifier); break; - case param_space_local: - printf( - "GPGPU-Sim PTX: allocating stack frame region for .param \"%s\" from " - "0x%x to 0x%lx\n", - identifier, g_current_symbol_table->get_local_next(), - g_current_symbol_table->get_local_next() + num_bits / 8); + case param_space_local: + printf("GPGPU-Sim PTX: allocating stack frame region for .param \"%s\" from 0x%x to 0x%lx\n", + identifier, + g_current_symbol_table->get_local_next(), + g_current_symbol_table->get_local_next() + num_bits/8 ); fflush(stdout); - assert((num_bits % 8) == 0); - g_last_symbol->set_address(g_current_symbol_table->get_local_next()); - g_current_symbol_table->alloc_local(num_bits / 8); - g_func_info->set_framesize(g_current_symbol_table->get_local_next()); + assert( (num_bits%8) == 0 ); + g_last_symbol->set_address( g_current_symbol_table->get_local_next() ); + g_current_symbol_table->alloc_local( num_bits/8 ); + g_func_info->set_framesize( g_current_symbol_table->get_local_next() ); break; - case param_space_kernel: + case param_space_kernel: break; - default: + default: abort(); break; - } + } - assert(!ti.is_param_unclassified()); - if (ti.is_param_kernel()) { - bool is_ptr = (g_ptr_spec != undefined_space); - g_func_info->add_param_name_type_size(g_entry_func_param_index, identifier, - ti.scalar_type(), num_bits, is_ptr, - g_ptr_spec); - g_entry_func_param_index++; - } + assert( !ti.is_param_unclassified() ); + if ( ti.is_param_kernel() ) { + bool is_ptr = (g_ptr_spec != undefined_space); + g_func_info->add_param_name_type_size(g_entry_func_param_index,identifier, ti.scalar_type(), num_bits, is_ptr, g_ptr_spec); + g_entry_func_param_index++; + } } -void ptx_recognizer::add_constptr(const char *identifier1, - const char *identifier2, int offset) { - symbol *s1 = g_current_symbol_table->lookup(identifier1); - const symbol *s2 = g_current_symbol_table->lookup(identifier2); - parse_assert(s1 != NULL, "'from' constant identifier does not exist."); - parse_assert(s1 != NULL, "'to' constant identifier does not exist."); +void ptx_recognizer::add_constptr(const char* identifier1, const char* identifier2, int offset) +{ + symbol *s1 = g_current_symbol_table->lookup(identifier1); + const symbol *s2 = g_current_symbol_table->lookup(identifier2); + parse_assert( s1 != NULL, "'from' constant identifier does not exist."); + parse_assert( s1 != NULL, "'to' constant identifier does not exist."); - unsigned addr = s2->get_address(); + unsigned addr = s2->get_address(); - printf("GPGPU-Sim PTX: moving \"%s\" from 0x%x to 0x%x (%s+%x)\n", - identifier1, s1->get_address(), addr + offset, identifier2, offset); + printf("GPGPU-Sim PTX: moving \"%s\" from 0x%x to 0x%x (%s+%x)\n", + identifier1, s1->get_address(), addr+offset, identifier2, offset); - s1->set_address(addr + offset); + s1->set_address( addr + offset ); } -void ptx_recognizer::add_function_arg() { - assert(g_size > 0); - if (g_func_info) { - PTX_PARSE_DPRINTF("add_function_arg \"%s\"", g_last_symbol->name().c_str()); - g_func_info->add_arg(g_last_symbol); - unsigned alignment = (g_alignment_spec == -1) ? g_size : g_alignment_spec; - assert(alignment == 1 || alignment == 2 || alignment == 4 || - alignment == 8 || alignment == 16); // known valid alignment values - g_func_info->add_config_param(g_size, alignment); - } +void ptx_recognizer::add_function_arg() +{ + assert(g_size>0); + if( g_func_info ) { + PTX_PARSE_DPRINTF("add_function_arg \"%s\"", g_last_symbol->name().c_str() ); + g_func_info->add_arg(g_last_symbol); + unsigned alignment = (g_alignment_spec==-1) ? g_size : g_alignment_spec; + assert(alignment==1||alignment==2||alignment==4||alignment==8||alignment==16);//known valid alignment values + g_func_info->add_config_param( g_size, alignment); + } + } -void ptx_recognizer::add_extern_spec() { - PTX_PARSE_DPRINTF("add_extern_spec"); - g_extern_spec = 1; +void ptx_recognizer::add_extern_spec() +{ + PTX_PARSE_DPRINTF("add_extern_spec"); + g_extern_spec = 1; } -void ptx_recognizer::add_alignment_spec(int spec) { - PTX_PARSE_DPRINTF("add_alignment_spec"); - parse_assert( - g_alignment_spec == -1, - "multiple .align specifiers per variable declaration not allowed."); - g_alignment_spec = spec; +void ptx_recognizer::add_alignment_spec( int spec ) +{ + PTX_PARSE_DPRINTF("add_alignment_spec"); + parse_assert( g_alignment_spec == -1, "multiple .align specifiers per variable declaration not allowed." ); + g_alignment_spec = spec; } -void ptx_recognizer::add_ptr_spec(enum _memory_space_t spec) { - PTX_PARSE_DPRINTF("add_ptr_spec \"%s\"", g_ptx_token_decode[spec].c_str()); - parse_assert(g_ptr_spec == undefined_space, - "multiple ptr space specifiers not allowed."); - parse_assert( - spec == global_space or spec == local_space or spec == shared_space, - "invalid space for ptr directive."); - g_ptr_spec = spec; +void ptx_recognizer::add_ptr_spec( enum _memory_space_t spec ) +{ + PTX_PARSE_DPRINTF("add_ptr_spec \"%s\"", g_ptx_token_decode[spec].c_str() ); + parse_assert( g_ptr_spec == undefined_space, "multiple ptr space specifiers not allowed." ); + parse_assert( spec == global_space or spec == local_space or spec == shared_space, "invalid space for ptr directive." ); + g_ptr_spec = spec; } -void ptx_recognizer::add_space_spec(enum _memory_space_t spec, int value) { - PTX_PARSE_DPRINTF("add_space_spec \"%s\"", g_ptx_token_decode[spec].c_str()); - parse_assert(g_space_spec == undefined_space, - "multiple space specifiers not allowed."); - if (spec == param_space_unclassified) { - if (g_func_decl) { - if (g_entry_point == 1) - g_space_spec = param_space_kernel; - else - g_space_spec = param_space_local; - } else - g_space_spec = param_space_unclassified; - } else { - g_space_spec = spec; - if (g_space_spec == const_space) g_space_spec.set_bank((unsigned)value); - } +void ptx_recognizer::add_space_spec( enum _memory_space_t spec, int value ) +{ + PTX_PARSE_DPRINTF("add_space_spec \"%s\"", g_ptx_token_decode[spec].c_str() ); + parse_assert( g_space_spec == undefined_space, "multiple space specifiers not allowed." ); + if( spec == param_space_unclassified ) { + if( g_func_decl ) { + if( g_entry_point == 1) + g_space_spec = param_space_kernel; + else + g_space_spec = param_space_local; + } else + g_space_spec = param_space_unclassified; + } else { + g_space_spec = spec; + if( g_space_spec == const_space ) + g_space_spec.set_bank((unsigned)value); + } } -void ptx_recognizer::add_vector_spec(int spec) { - PTX_PARSE_DPRINTF("add_vector_spec"); - parse_assert(g_vector_spec == -1, "multiple vector specifiers not allowed."); - g_vector_spec = spec; +void ptx_recognizer::add_vector_spec(int spec ) +{ + PTX_PARSE_DPRINTF("add_vector_spec"); + parse_assert( g_vector_spec == -1, "multiple vector specifiers not allowed." ); + g_vector_spec = spec; } -void ptx_recognizer::add_scalar_type_spec(int type_spec) { - // save size of parameter - switch (type_spec) { - case B8_TYPE: - case S8_TYPE: - case U8_TYPE: - g_size = 1; - break; - case B16_TYPE: - case S16_TYPE: - case U16_TYPE: - case F16_TYPE: - g_size = 2; - break; - case B32_TYPE: - case S32_TYPE: - case U32_TYPE: - case F32_TYPE: - g_size = 4; - break; - case B64_TYPE: - case BB64_TYPE: - case S64_TYPE: - case U64_TYPE: - case F64_TYPE: - case FF64_TYPE: - g_size = 8; - break; - case BB128_TYPE: - g_size = 16; - break; - } - PTX_PARSE_DPRINTF("add_scalar_type_spec \"%s\"", - g_ptx_token_decode[type_spec].c_str()); - g_scalar_type.push_back(type_spec); - if (g_scalar_type.size() > 1) { - parse_assert((g_opcode == -1) || (g_opcode == CVT_OP) || - (g_opcode == SET_OP) || (g_opcode == SLCT_OP) || - (g_opcode == TEX_OP) || (g_opcode == MMA_OP) || - (g_opcode == DP4A_OP), - "only cvt, set, slct, tex and dp4a can have more than one " - "type specifier."); - } - g_scalar_type_spec = type_spec; +void ptx_recognizer::add_scalar_type_spec( int type_spec ) +{ + //save size of parameter + switch ( type_spec ) { + case B8_TYPE: + case S8_TYPE: + case U8_TYPE: + g_size = 1; break; + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: + case F16_TYPE: + g_size = 2; break; + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: + case F32_TYPE: + g_size = 4; break; + case B64_TYPE: + case BB64_TYPE: + case S64_TYPE: + case U64_TYPE: + case F64_TYPE: + case FF64_TYPE: + g_size = 8; break; + case BB128_TYPE: + g_size = 16; break; + } + PTX_PARSE_DPRINTF("add_scalar_type_spec \"%s\"", g_ptx_token_decode[type_spec].c_str()); + g_scalar_type.push_back( type_spec ); + if ( g_scalar_type.size() > 1 ) { + parse_assert( (g_opcode == -1) || (g_opcode == CVT_OP) || (g_opcode == SET_OP) || (g_opcode == SLCT_OP) + || (g_opcode == TEX_OP)|| (g_opcode==MMA_OP)|| (g_opcode == DP4A_OP), + "only cvt, set, slct, tex and dp4a can have more than one type specifier."); + } + g_scalar_type_spec = type_spec; } -void ptx_recognizer::add_label(const char *identifier) { - PTX_PARSE_DPRINTF("add_label"); - symbol *s = g_current_symbol_table->lookup(identifier); - if (s != NULL) { - g_label = s; - } else { - g_label = g_current_symbol_table->add_variable( - identifier, NULL, 0, gpgpu_ctx->g_filename, ptx_get_lineno(scanner)); - } +void ptx_recognizer::add_label( const char *identifier ) +{ + PTX_PARSE_DPRINTF("add_label"); + symbol *s = g_current_symbol_table->lookup(identifier); + if ( s != NULL ) { + g_label = s; + } else { + g_label = g_current_symbol_table->add_variable(identifier,NULL,0,gpgpu_ctx->g_filename,ptx_get_lineno(scanner)); + } } -void ptx_recognizer::add_opcode(int opcode) { g_opcode = opcode; } +void ptx_recognizer::add_opcode( int opcode ) +{ + g_opcode = opcode; +} -void ptx_recognizer::add_pred(const char *identifier, int neg, - int predModifier) { - PTX_PARSE_DPRINTF("add_pred"); - const symbol *s = g_current_symbol_table->lookup(identifier); - if (s == NULL) { - std::string msg = - std::string("predicate \"") + identifier + "\" has no declaration."; - parse_error(msg.c_str()); - } - g_pred = s; - g_neg_pred = neg; - g_pred_mod = predModifier; +void ptx_recognizer::add_pred( const char *identifier, int neg, int predModifier ) +{ + PTX_PARSE_DPRINTF("add_pred"); + const symbol *s = g_current_symbol_table->lookup(identifier); + if ( s == NULL ) { + std::string msg = std::string("predicate \"") + identifier + "\" has no declaration."; + parse_error( msg.c_str() ); + } + g_pred = s; + g_neg_pred = neg; + g_pred_mod = predModifier; } -void ptx_recognizer::add_option(int option) { - PTX_PARSE_DPRINTF("add_option"); - g_options.push_back(option); +void ptx_recognizer::add_option( int option ) +{ + PTX_PARSE_DPRINTF("add_option"); + g_options.push_back( option ); } -void ptx_recognizer::add_wmma_option(int option) { - PTX_PARSE_DPRINTF("add_option"); - g_wmma_options.push_back(option); +void ptx_recognizer::add_wmma_option( int option ) +{ + PTX_PARSE_DPRINTF("add_option"); + g_wmma_options.push_back( option ); } -void ptx_recognizer::add_double_operand(const char *d1, const char *d2) { - // operands that access two variables. - // eg. s[$ofs1+$r0], g[$ofs1+=$r0] - // TODO: Not sure if I'm going to use this for storing to two destinations or - // not. +void ptx_recognizer::add_double_operand( const char *d1, const char *d2 ) +{ + //operands that access two variables. + //eg. s[$ofs1+$r0], g[$ofs1+=$r0] + //TODO: Not sure if I'm going to use this for storing to two destinations or not. - PTX_PARSE_DPRINTF("add_double_operand"); - const symbol *s1 = g_current_symbol_table->lookup(d1); - const symbol *s2 = g_current_symbol_table->lookup(d2); - parse_assert(s1 != NULL && s2 != NULL, "component(s) missing declarations."); - g_operands.push_back(operand_info(s1, s2, gpgpu_ctx)); + PTX_PARSE_DPRINTF("add_double_operand"); + const symbol *s1 = g_current_symbol_table->lookup(d1); + const symbol *s2 = g_current_symbol_table->lookup(d2); + parse_assert( s1 != NULL && s2 != NULL, "component(s) missing declarations."); + g_operands.push_back( operand_info(s1,s2,gpgpu_ctx) ); } -void ptx_recognizer::add_1vector_operand(const char *d1) { - // handles the single element vector operand ({%v1}) found in tex.1d - // instructions - PTX_PARSE_DPRINTF("add_1vector_operand"); - const symbol *s1 = g_current_symbol_table->lookup(d1); - parse_assert(s1 != NULL, "component(s) missing declarations."); - g_operands.push_back(operand_info(s1, NULL, NULL, NULL, gpgpu_ctx)); +void ptx_recognizer::add_1vector_operand( const char *d1 ) +{ + // handles the single element vector operand ({%v1}) found in tex.1d instructions + PTX_PARSE_DPRINTF("add_1vector_operand"); + const symbol *s1 = g_current_symbol_table->lookup(d1); + parse_assert( s1 != NULL, "component(s) missing declarations."); + g_operands.push_back( operand_info(s1,NULL,NULL,NULL,gpgpu_ctx) ); } -void ptx_recognizer::add_2vector_operand(const char *d1, const char *d2) { - PTX_PARSE_DPRINTF("add_2vector_operand"); - const symbol *s1 = g_current_symbol_table->lookup(d1); - const symbol *s2 = g_current_symbol_table->lookup(d2); - parse_assert(s1 != NULL && s2 != NULL, - "v2 component(s) missing declarations."); - g_operands.push_back(operand_info(s1, s2, NULL, NULL, gpgpu_ctx)); +void ptx_recognizer::add_2vector_operand( const char *d1, const char *d2 ) +{ + PTX_PARSE_DPRINTF("add_2vector_operand"); + const symbol *s1 = g_current_symbol_table->lookup(d1); + const symbol *s2 = g_current_symbol_table->lookup(d2); + parse_assert( s1 != NULL && s2 != NULL, "v2 component(s) missing declarations."); + g_operands.push_back( operand_info(s1,s2,NULL,NULL,gpgpu_ctx) ); } -void ptx_recognizer::add_3vector_operand(const char *d1, const char *d2, - const char *d3) { - PTX_PARSE_DPRINTF("add_3vector_operand"); - const symbol *s1 = g_current_symbol_table->lookup(d1); - const symbol *s2 = g_current_symbol_table->lookup(d2); - const symbol *s3 = g_current_symbol_table->lookup(d3); - parse_assert(s1 != NULL && s2 != NULL && s3 != NULL, - "v3 component(s) missing declarations."); - g_operands.push_back(operand_info(s1, s2, s3, NULL, gpgpu_ctx)); +void ptx_recognizer::add_3vector_operand( const char *d1, const char *d2, const char *d3 ) +{ + PTX_PARSE_DPRINTF("add_3vector_operand"); + const symbol *s1 = g_current_symbol_table->lookup(d1); + const symbol *s2 = g_current_symbol_table->lookup(d2); + const symbol *s3 = g_current_symbol_table->lookup(d3); + parse_assert( s1 != NULL && s2 != NULL && s3 != NULL, "v3 component(s) missing declarations."); + g_operands.push_back( operand_info(s1,s2,s3,NULL,gpgpu_ctx) ); } -void ptx_recognizer::add_4vector_operand(const char *d1, const char *d2, - const char *d3, const char *d4) { - PTX_PARSE_DPRINTF("add_4vector_operand"); - const symbol *s1 = g_current_symbol_table->lookup(d1); - const symbol *s2 = g_current_symbol_table->lookup(d2); - const symbol *s3 = g_current_symbol_table->lookup(d3); - const symbol *s4 = g_current_symbol_table->lookup(d4); - parse_assert(s1 != NULL && s2 != NULL && s3 != NULL && s4 != NULL, - "v4 component(s) missing declarations."); - const symbol *null_op = g_current_symbol_table->lookup("_"); - if (s2 == null_op) s2 = NULL; - if (s3 == null_op) s3 = NULL; - if (s4 == null_op) s4 = NULL; - g_operands.push_back(operand_info(s1, s2, s3, s4, gpgpu_ctx)); +void ptx_recognizer::add_4vector_operand( const char *d1, const char *d2, const char *d3, const char *d4 ) +{ + PTX_PARSE_DPRINTF("add_4vector_operand"); + const symbol *s1 = g_current_symbol_table->lookup(d1); + const symbol *s2 = g_current_symbol_table->lookup(d2); + const symbol *s3 = g_current_symbol_table->lookup(d3); + const symbol *s4 = g_current_symbol_table->lookup(d4); + parse_assert( s1 != NULL && s2 != NULL && s3 != NULL && s4 != NULL, "v4 component(s) missing declarations."); + const symbol *null_op = g_current_symbol_table->lookup("_"); + if ( s2 == null_op ) s2 = NULL; + if ( s3 == null_op ) s3 = NULL; + if ( s4 == null_op ) s4 = NULL; + g_operands.push_back( operand_info(s1,s2,s3,s4,gpgpu_ctx) ); } -void ptx_recognizer::add_8vector_operand(const char *d1, const char *d2, - const char *d3, const char *d4, - const char *d5, const char *d6, - const char *d7, const char *d8) { - PTX_PARSE_DPRINTF("add_8vector_operand"); - const symbol *s1 = g_current_symbol_table->lookup(d1); - const symbol *s2 = g_current_symbol_table->lookup(d2); - const symbol *s3 = g_current_symbol_table->lookup(d3); - const symbol *s4 = g_current_symbol_table->lookup(d4); - const symbol *s5 = g_current_symbol_table->lookup(d5); - const symbol *s6 = g_current_symbol_table->lookup(d6); - const symbol *s7 = g_current_symbol_table->lookup(d7); - const symbol *s8 = g_current_symbol_table->lookup(d8); - parse_assert(s1 != NULL && s2 != NULL && s3 != NULL && s4 != NULL && - s5 != NULL && s6 != NULL && s7 != NULL && s8 != NULL, - "v4 component(s) missing declarations."); - const symbol *null_op = g_current_symbol_table->lookup("_"); - if (s2 == null_op) s2 = NULL; - if (s3 == null_op) s3 = NULL; - if (s4 == null_op) s4 = NULL; - if (s5 == null_op) s5 = NULL; - if (s6 == null_op) s6 = NULL; - if (s7 == null_op) s7 = NULL; - if (s8 == null_op) s8 = NULL; - g_operands.push_back(operand_info(s1, s2, s3, s4, s5, s6, s7, s8, gpgpu_ctx)); +void ptx_recognizer::add_8vector_operand( const char *d1, const char *d2, const char *d3, const char *d4,const char *d5,const char *d6,const char *d7,const char *d8 ) +{ + PTX_PARSE_DPRINTF("add_8vector_operand"); + const symbol *s1 = g_current_symbol_table->lookup(d1); + const symbol *s2 = g_current_symbol_table->lookup(d2); + const symbol *s3 = g_current_symbol_table->lookup(d3); + const symbol *s4 = g_current_symbol_table->lookup(d4); + const symbol *s5 = g_current_symbol_table->lookup(d5); + const symbol *s6 = g_current_symbol_table->lookup(d6); + const symbol *s7 = g_current_symbol_table->lookup(d7); + const symbol *s8 = g_current_symbol_table->lookup(d8); + parse_assert( s1 != NULL && s2 != NULL && s3 != NULL && s4 != NULL && s5 !=NULL && s6 !=NULL && s7 !=NULL && s8 !=NULL, "v4 component(s) missing declarations."); + const symbol *null_op = g_current_symbol_table->lookup("_"); + if ( s2 == null_op ) s2 = NULL; + if ( s3 == null_op ) s3 = NULL; + if ( s4 == null_op ) s4 = NULL; + if ( s5 == null_op ) s5 = NULL; + if ( s6 == null_op ) s6 = NULL; + if ( s7 == null_op ) s7 = NULL; + if ( s8 == null_op ) s8 = NULL; + g_operands.push_back( operand_info(s1,s2,s3,s4,s5,s6,s7,s8,gpgpu_ctx) ); } -void ptx_recognizer::add_builtin_operand(int builtin, int dim_modifier) { - PTX_PARSE_DPRINTF("add_builtin_operand"); - g_operands.push_back(operand_info(builtin, dim_modifier, gpgpu_ctx)); +void ptx_recognizer::add_builtin_operand( int builtin, int dim_modifier ) +{ + PTX_PARSE_DPRINTF("add_builtin_operand"); + g_operands.push_back( operand_info(builtin,dim_modifier,gpgpu_ctx) ); } -void ptx_recognizer::add_memory_operand() { - PTX_PARSE_DPRINTF("add_memory_operand"); - assert(!g_operands.empty()); - g_operands.back().make_memory_operand(); +void ptx_recognizer::add_memory_operand() +{ + PTX_PARSE_DPRINTF("add_memory_operand"); + assert( !g_operands.empty() ); + g_operands.back().make_memory_operand(); } /*TODO: add other memory locations*/ -void ptx_recognizer::change_memory_addr_space(const char *identifier) { - /*0 = N/A, not reading from memory - *1 = global memory - *2 = shared memory - *3 = const memory segment - *4 = local memory segment - */ +void ptx_recognizer::change_memory_addr_space(const char *identifier) +{ + /*0 = N/A, not reading from memory + *1 = global memory + *2 = shared memory + *3 = const memory segment + *4 = local memory segment + */ - bool recognizedType = false; + bool recognizedType = false; - PTX_PARSE_DPRINTF("change_memory_addr_space"); - assert(!g_operands.empty()); - if (!strcmp(identifier, "g")) { - g_operands.back().set_addr_space(global_space); - recognizedType = true; - } - if (!strcmp(identifier, "s")) { - g_operands.back().set_addr_space(shared_space); - recognizedType = true; - } - // For constants, check if the first character is 'c' - char c[2]; - strncpy(c, identifier, 1); - c[1] = '\0'; - if (!strcmp(c, "c")) { - g_operands.back().set_addr_space(const_space); - parse_assert(g_current_symbol_table->lookup(identifier) != NULL, - "Constant was not defined."); - g_operands.back().set_const_mem_offset( - g_current_symbol_table->lookup(identifier)->get_address()); - recognizedType = true; - } - // For local memory, check if the first character is 'l' - char l[2]; - strncpy(l, identifier, 1); - l[1] = '\0'; - if (!strcmp(l, "l")) { - g_operands.back().set_addr_space(local_space); - // parse_assert(g_current_symbol_table->lookup(identifier) != NULL, "Local - // memory segment was not defined."); - // g_operands.back().set_const_mem_offset(g_current_symbol_table->lookup(identifier)->get_address()); - recognizedType = true; - } + PTX_PARSE_DPRINTF("change_memory_addr_space"); + assert( !g_operands.empty() ); + if(!strcmp(identifier, "g")) + { + g_operands.back().set_addr_space(global_space); + recognizedType = true; + } + if(!strcmp(identifier, "s")) + { + g_operands.back().set_addr_space(shared_space); + recognizedType = true; + } + // For constants, check if the first character is 'c' + char c[2]; + strncpy(c, identifier, 1); c[1] = '\0'; + if(!strcmp(c, "c")) + { + g_operands.back().set_addr_space(const_space); + parse_assert(g_current_symbol_table->lookup(identifier) != NULL, "Constant was not defined."); + g_operands.back().set_const_mem_offset(g_current_symbol_table->lookup(identifier)->get_address()); + recognizedType = true; + } + // For local memory, check if the first character is 'l' + char l[2]; + strncpy(l, identifier, 1); l[1] = '\0'; + if(!strcmp(l, "l")) + { + g_operands.back().set_addr_space(local_space); + //parse_assert(g_current_symbol_table->lookup(identifier) != NULL, "Local memory segment was not defined."); + //g_operands.back().set_const_mem_offset(g_current_symbol_table->lookup(identifier)->get_address()); + recognizedType = true; + } - parse_assert(recognizedType, "Error: unrecognized memory type."); + parse_assert(recognizedType, "Error: unrecognized memory type."); } -void ptx_recognizer::change_operand_lohi(int lohi) { - /*0 = N/A, read entire operand - *1 = lo, reading from lowest bits - *2 = hi, reading from highest bits - */ +void ptx_recognizer::change_operand_lohi( int lohi ) +{ + /*0 = N/A, read entire operand + *1 = lo, reading from lowest bits + *2 = hi, reading from highest bits + */ - PTX_PARSE_DPRINTF("change_operand_lohi"); - assert(!g_operands.empty()); + PTX_PARSE_DPRINTF("change_operand_lohi"); + assert( !g_operands.empty() ); + + g_operands.back().set_operand_lohi(lohi); - g_operands.back().set_operand_lohi(lohi); } -void ptx_recognizer::set_immediate_operand_type() { - PTX_PARSE_DPRINTF("set_immediate_operand_type"); - assert(!g_operands.empty()); - g_operands.back().set_immediate_addr(); +void ptx_recognizer::set_immediate_operand_type () +{ + PTX_PARSE_DPRINTF("set_immediate_operand_type"); + assert( !g_operands.empty() ); + g_operands.back().set_immediate_addr(); } -void ptx_recognizer::change_double_operand_type(int operand_type) { - /* - *-3 = reg / reg (set instruction, but both get same value) - *-2 = reg | reg (cvt instruction) - *-1 = reg | reg (set instruction) - *0 = N/A, default - *1 = reg + reg - *2 = reg += reg - *3 = reg += immediate - */ +void ptx_recognizer::change_double_operand_type( int operand_type ) +{ + /* + *-3 = reg / reg (set instruction, but both get same value) + *-2 = reg | reg (cvt instruction) + *-1 = reg | reg (set instruction) + *0 = N/A, default + *1 = reg + reg + *2 = reg += reg + *3 = reg += immediate + */ - PTX_PARSE_DPRINTF("change_double_operand_type"); - assert(!g_operands.empty()); + PTX_PARSE_DPRINTF("change_double_operand_type"); + assert( !g_operands.empty() ); - // For double destination operands, ensure valid instruction - if (operand_type == -1 || operand_type == -2) { - if ((g_opcode == SET_OP) || (g_opcode == SETP_OP)) - g_operands.back().set_double_operand_type(-1); - else - g_operands.back().set_double_operand_type(-2); - } else if (operand_type == -3) { - if (g_opcode == SET_OP || g_opcode == MAD_OP) + // For double destination operands, ensure valid instruction + if( operand_type == -1 || operand_type == -2 ) { + if((g_opcode == SET_OP)||(g_opcode == SETP_OP)) + g_operands.back().set_double_operand_type(-1); + else + g_operands.back().set_double_operand_type(-2); + } else if( operand_type == -3 ) { + if(g_opcode == SET_OP || g_opcode == MAD_OP) + g_operands.back().set_double_operand_type(operand_type); + else + parse_assert(0, "Error: Unsupported use of double destination operand."); + } else { g_operands.back().set_double_operand_type(operand_type); - else - parse_assert(0, "Error: Unsupported use of double destination operand."); - } else { - g_operands.back().set_double_operand_type(operand_type); - } + } + } -void ptx_recognizer::change_operand_neg() { - PTX_PARSE_DPRINTF("change_operand_neg"); - assert(!g_operands.empty()); +void ptx_recognizer::change_operand_neg( ) +{ + PTX_PARSE_DPRINTF("change_operand_neg"); + assert( !g_operands.empty() ); + + g_operands.back().set_operand_neg(); - g_operands.back().set_operand_neg(); } -void ptx_recognizer::add_literal_int(int value) { - PTX_PARSE_DPRINTF("add_literal_int"); - g_operands.push_back(operand_info(value, gpgpu_ctx)); +void ptx_recognizer::add_literal_int( int value ) +{ + PTX_PARSE_DPRINTF("add_literal_int"); + g_operands.push_back( operand_info(value,gpgpu_ctx) ); } -void ptx_recognizer::add_literal_float(float value) { - PTX_PARSE_DPRINTF("add_literal_float"); - g_operands.push_back(operand_info(value, gpgpu_ctx)); +void ptx_recognizer::add_literal_float( float value ) +{ + PTX_PARSE_DPRINTF("add_literal_float"); + g_operands.push_back( operand_info(value,gpgpu_ctx) ); } -void ptx_recognizer::add_literal_double(double value) { - PTX_PARSE_DPRINTF("add_literal_double"); - g_operands.push_back(operand_info(value, gpgpu_ctx)); +void ptx_recognizer::add_literal_double( double value ) +{ + PTX_PARSE_DPRINTF("add_literal_double"); + g_operands.push_back( operand_info(value,gpgpu_ctx) ); } -void ptx_recognizer::add_scalar_operand(const char *identifier) { - PTX_PARSE_DPRINTF("add_scalar_operand"); - const symbol *s = g_current_symbol_table->lookup(identifier); - if (s == NULL) { - if (g_opcode == BRA_OP || g_opcode == CALLP_OP) { - // forward branch target... - s = g_current_symbol_table->add_variable( - identifier, NULL, 0, gpgpu_ctx->g_filename, ptx_get_lineno(scanner)); - } else { - std::string msg = - std::string("operand \"") + identifier + "\" has no declaration."; - parse_error(msg.c_str()); - } - } - g_operands.push_back(operand_info(s, gpgpu_ctx)); +void ptx_recognizer::add_scalar_operand( const char *identifier ) +{ + PTX_PARSE_DPRINTF("add_scalar_operand"); + const symbol *s = g_current_symbol_table->lookup(identifier); + if ( s == NULL ) { + if ( g_opcode == BRA_OP || g_opcode == CALLP_OP) { + // forward branch target... + s = g_current_symbol_table->add_variable(identifier,NULL,0,gpgpu_ctx->g_filename,ptx_get_lineno(scanner)); + } else { + std::string msg = std::string("operand \"") + identifier + "\" has no declaration."; + parse_error( msg.c_str() ); + } + } + g_operands.push_back( operand_info(s,gpgpu_ctx) ); } -void ptx_recognizer::add_neg_pred_operand(const char *identifier) { - PTX_PARSE_DPRINTF("add_neg_pred_operand"); - const symbol *s = g_current_symbol_table->lookup(identifier); - if (s == NULL) { - s = g_current_symbol_table->add_variable( - identifier, NULL, 1, gpgpu_ctx->g_filename, ptx_get_lineno(scanner)); - } - operand_info op(s, gpgpu_ctx); - op.set_neg_pred(); - g_operands.push_back(op); +void ptx_recognizer::add_neg_pred_operand( const char *identifier ) +{ + PTX_PARSE_DPRINTF("add_neg_pred_operand"); + const symbol *s = g_current_symbol_table->lookup(identifier); + if ( s == NULL ) { + s = g_current_symbol_table->add_variable(identifier,NULL,1,gpgpu_ctx->g_filename,ptx_get_lineno(scanner)); + } + operand_info op(s, gpgpu_ctx); + op.set_neg_pred(); + g_operands.push_back( op ); } -void ptx_recognizer::add_address_operand(const char *identifier, int offset) { - PTX_PARSE_DPRINTF("add_address_operand"); - const symbol *s = g_current_symbol_table->lookup(identifier); - if (s == NULL) { - std::string msg = - std::string("operand \"") + identifier + "\" has no declaration."; - parse_error(msg.c_str()); - } - g_operands.push_back(operand_info(s, offset, gpgpu_ctx)); +void ptx_recognizer::add_address_operand( const char *identifier, int offset ) +{ + PTX_PARSE_DPRINTF("add_address_operand"); + const symbol *s = g_current_symbol_table->lookup(identifier); + if ( s == NULL ) { + std::string msg = std::string("operand \"") + identifier + "\" has no declaration."; + parse_error( msg.c_str() ); + } + g_operands.push_back( operand_info(s,offset,gpgpu_ctx) ); } -void ptx_recognizer::add_address_operand2(int offset) { - PTX_PARSE_DPRINTF("add_address_operand"); - g_operands.push_back(operand_info((unsigned)offset, gpgpu_ctx)); +void ptx_recognizer::add_address_operand2( int offset ) +{ + PTX_PARSE_DPRINTF("add_address_operand"); + g_operands.push_back( operand_info((unsigned)offset,gpgpu_ctx) ); } -void ptx_recognizer::add_array_initializer() { - g_last_symbol->add_initializer(g_operands); +void ptx_recognizer::add_array_initializer() +{ + g_last_symbol->add_initializer(g_operands); } -void ptx_recognizer::add_version_info(float ver, unsigned ext) { - g_global_symbol_table->set_ptx_version(ver, ext); +void ptx_recognizer::add_version_info( float ver, unsigned ext ) +{ + g_global_symbol_table->set_ptx_version(ver,ext); } -void ptx_recognizer::add_file(unsigned num, const char *filename) { - if (gpgpu_ctx->g_filename == NULL) { - char *b = strdup(filename); - char *l = b; - char *n = b; - while (*n != '\0') { - if (*n == '/') l = n + 1; - n++; - } +void ptx_recognizer::add_file( unsigned num, const char *filename ) +{ + if( gpgpu_ctx->g_filename == NULL ) { + char *b = strdup(filename); + char *l=b; + char *n=b; + while( *n != '\0' ) { + if( *n == '/' ) + l = n+1; + n++; + } - char *p = strtok(l, "."); - char buf[1024]; - snprintf(buf, 1024, "%s.ptx", p); + char *p = strtok(l,"."); + char buf[1024]; + snprintf(buf,1024,"%s.ptx",p); - char *q = strtok(NULL, "."); - if (q && !strcmp(q, "cu")) { - gpgpu_ctx->g_filename = strdup(buf); - } + char *q = strtok(NULL,"."); + if( q && !strcmp(q,"cu") ) { + gpgpu_ctx->g_filename = strdup(buf); + } - free(b); - } + free( b ); + } - g_current_symbol_table = g_global_symbol_table; + g_current_symbol_table = g_global_symbol_table; } -void *ptx_recognizer::reset_symtab() { - void *result = g_current_symbol_table; - g_current_symbol_table = g_global_symbol_table; - return result; +void * ptx_recognizer::reset_symtab() +{ + void *result = g_current_symbol_table; + g_current_symbol_table = g_global_symbol_table; + return result; } -void ptx_recognizer::set_symtab(void *symtab) { - g_current_symbol_table = (symbol_table *)symtab; +void ptx_recognizer::set_symtab(void*symtab) +{ + g_current_symbol_table = (symbol_table*)symtab; } -void ptx_recognizer::add_pragma(const char *str) { - printf("GPGPU-Sim PTX: Warning -- ignoring pragma '%s'\n", str); +void ptx_recognizer::add_pragma( const char *str ) +{ + printf("GPGPU-Sim PTX: Warning -- ignoring pragma '%s'\n", str ); } -void ptx_recognizer::version_header(double a) {} // intentional dummy function +void ptx_recognizer::version_header(double a) {} //intentional dummy function -void ptx_recognizer::target_header(char *a) { - g_global_symbol_table->set_sm_target(a, NULL, NULL); +void ptx_recognizer::target_header(char* a) +{ + g_global_symbol_table->set_sm_target(a,NULL,NULL); } -void ptx_recognizer::target_header2(char *a, char *b) { - g_global_symbol_table->set_sm_target(a, b, NULL); +void ptx_recognizer::target_header2(char* a, char* b) +{ + g_global_symbol_table->set_sm_target(a,b,NULL); } -void ptx_recognizer::target_header3(char *a, char *b, char *c) { - g_global_symbol_table->set_sm_target(a, b, c); +void ptx_recognizer::target_header3(char* a, char* b, char* c) +{ + g_global_symbol_table->set_sm_target(a,b,c); } void ptx_recognizer::maxnt_id(int x, int y, int z) { g_func_info->set_maxnt_id(x * y * z); } -void ptx_recognizer::func_header(const char *a) {} // intentional dummy - // function -void ptx_recognizer::func_header_info(const char *a) { -} // intentional dummy function -void ptx_recognizer::func_header_info_int(const char *a, int b) { -} // intentional dummy function +void ptx_recognizer::func_header(const char* a) {} //intentional dummy function +void ptx_recognizer::func_header_info(const char* a) {} //intentional dummy function +void ptx_recognizer::func_header_info_int(const char* a, int b) {} //intentional dummy function diff --git a/src/cuda-sim/ptx_parser.h b/src/cuda-sim/ptx_parser.h index 1e1b780..11a3d20 100644 --- a/src/cuda-sim/ptx_parser.h +++ b/src/cuda-sim/ptx_parser.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -34,161 +32,155 @@ #include "ptx_ir.h" class gpgpu_context; -typedef void *yyscan_t; +typedef void * yyscan_t; class ptx_recognizer { - public: - ptx_recognizer(gpgpu_context *ctx) : g_return_var(ctx) { - scanner = NULL; - g_size = -1; - g_add_identifier_cached__identifier = NULL; - g_alignment_spec = -1; - g_var_type = NULL; - g_opcode = -1; - g_space_spec = undefined_space; - g_ptr_spec = undefined_space; - g_scalar_type_spec = -1; - g_vector_spec = -1; - g_extern_spec = 0; - g_func_decl = 0; - g_ident_add_uid = 0; - g_const_alloc = 1; - g_max_regs_per_thread = 0; - g_global_symbol_table = NULL; - g_current_symbol_table = NULL; - g_last_symbol = NULL; - g_error_detected = 0; - g_entry_func_param_index = 0; - g_func_info = NULL; - g_debug_ir_generation = false; - gpgpu_ctx = ctx; - } - // global list - yyscan_t scanner; -#define PTX_LINEBUF_SIZE (4 * 1024) - char linebuf[PTX_LINEBUF_SIZE]; - unsigned col; - int g_size; - char *g_add_identifier_cached__identifier; - int g_add_identifier_cached__array_dim; - int g_add_identifier_cached__array_ident; - int g_alignment_spec; - // variable declaration stuff: - type_info *g_var_type; - // instruction definition stuff: - const symbol *g_pred; - int g_neg_pred; - int g_pred_mod; - symbol *g_label; - int g_opcode; - std::list<operand_info> g_operands; - std::list<int> g_options; - std::list<int> g_wmma_options; - std::list<int> g_scalar_type; - // type specifier stuff: - memory_space_t g_space_spec; - memory_space_t g_ptr_spec; - int g_scalar_type_spec; - int g_vector_spec; - int g_extern_spec; - int g_func_decl; - int g_ident_add_uid; - unsigned g_const_alloc; - unsigned g_max_regs_per_thread; - symbol_table *g_global_symbol_table; - symbol_table *g_current_symbol_table; - symbol *g_last_symbol; - std::list<ptx_instruction *> g_instructions; - int g_error_detected; - unsigned g_entry_func_param_index; - function_info *g_func_info; - operand_info g_return_var; - bool g_debug_ir_generation; - int g_entry_point; - const struct core_config *g_shader_core_config; - std::map<std::string, std::map<unsigned, const ptx_instruction *> > - g_inst_lookup; - // the program intermediate representation... - std::map<std::string, symbol_table *> g_sym_name_to_symbol_table; - // backward pointer - class gpgpu_context *gpgpu_ctx; + public: + ptx_recognizer( gpgpu_context* ctx ) : g_return_var(ctx) { + scanner = NULL; + g_size = -1; + g_add_identifier_cached__identifier = NULL; + g_alignment_spec = -1; + g_var_type = NULL; + g_opcode = -1; + g_space_spec = undefined_space; + g_ptr_spec = undefined_space; + g_scalar_type_spec = -1; + g_vector_spec = -1; + g_extern_spec = 0; + g_func_decl = 0; + g_ident_add_uid = 0; + g_const_alloc = 1; + g_max_regs_per_thread = 0; + g_global_symbol_table = NULL; + g_current_symbol_table = NULL; + g_last_symbol = NULL; + g_error_detected = 0; + g_entry_func_param_index=0; + g_func_info = NULL; + g_debug_ir_generation=false; + gpgpu_ctx = ctx; + } + // global list + yyscan_t scanner; +#define PTX_LINEBUF_SIZE (4*1024) + char linebuf[PTX_LINEBUF_SIZE]; + unsigned col; + int g_size; + char *g_add_identifier_cached__identifier; + int g_add_identifier_cached__array_dim; + int g_add_identifier_cached__array_ident; + int g_alignment_spec; + // variable declaration stuff: + type_info *g_var_type; + // instruction definition stuff: + const symbol *g_pred; + int g_neg_pred; + int g_pred_mod; + symbol *g_label; + int g_opcode; + std::list<operand_info> g_operands; + std::list<int> g_options; + std::list<int> g_wmma_options; + std::list<int> g_scalar_type; + // type specifier stuff: + memory_space_t g_space_spec; + memory_space_t g_ptr_spec; + int g_scalar_type_spec; + int g_vector_spec; + int g_extern_spec; + int g_func_decl; + int g_ident_add_uid; + unsigned g_const_alloc; + unsigned g_max_regs_per_thread; + symbol_table *g_global_symbol_table; + symbol_table *g_current_symbol_table; + symbol *g_last_symbol; + std::list<ptx_instruction*> g_instructions; + int g_error_detected; + unsigned g_entry_func_param_index; + function_info *g_func_info; + operand_info g_return_var; + bool g_debug_ir_generation; + int g_entry_point; + const struct core_config *g_shader_core_config; + std::map<std::string,std::map<unsigned,const ptx_instruction*> > g_inst_lookup; + // the program intermediate representation... + std::map<std::string,symbol_table*> g_sym_name_to_symbol_table; + // backward pointer + class gpgpu_context* gpgpu_ctx; + + // member function list + void init_directive_state(); + void init_instruction_state(); + void start_function( int entry_point ); + void add_function_name( const char *fname ); + void add_directive(); + void end_function(); + void add_identifier( const char *s, int array_dim, unsigned array_ident ); + void add_function_arg(); + void add_scalar_type_spec( int type_spec ); + void add_scalar_operand( const char *identifier ); + void add_neg_pred_operand( const char *identifier ); + void add_variables(); + void set_variable_type(); + void add_opcode( int opcode ); + void add_pred( const char *identifier, int negate, int predModifier ); + void add_1vector_operand( const char *d1 ); + void add_2vector_operand( const char *d1, const char *d2 ); + void add_3vector_operand( const char *d1, const char *d2, const char *d3 ); + void add_4vector_operand( const char *d1, const char *d2, const char *d3, const char *d4 ); + void add_8vector_operand( const char *d1, const char *d2, const char *d3, const char *d4 ,const char *d5,const char *d6,const char *d7,const char *d8); + void add_option(int option ); + void add_wmma_option(int option ); + void add_builtin_operand( int builtin, int dim_modifier ); + void add_memory_operand( ); + void add_literal_int( int value ); + void add_literal_float( float value ); + void add_literal_double( double value ); + void add_address_operand( const char *identifier, int offset ); + void add_address_operand2( int offset ); + void add_label( const char *idenfiier ); + void add_vector_spec(int spec ); + void add_space_spec( enum _memory_space_t spec, int value ); + void add_ptr_spec( enum _memory_space_t spec ); + void add_extern_spec(); + void add_instruction(); + void set_return(); + void add_alignment_spec( int spec ); + void add_array_initializer(); + void add_file( unsigned num, const char *filename ); + void add_version_info( float ver, unsigned ext); + void *reset_symtab(); + void set_symtab(void*); + void add_pragma( const char *str ); + void func_header(const char* a); + void func_header_info(const char* a); + void func_header_info_int(const char* a, int b); + void add_constptr(const char* identifier1, const char* identifier2, int offset); + void target_header(char* a); + void target_header2(char* a, char* b); + void target_header3(char* a, char* b, char* c); + void add_double_operand( const char *d1, const char *d2 ); + void change_memory_addr_space( const char *identifier ); + void change_operand_lohi( int lohi ); + void change_double_operand_type( int addr_type ); + void change_operand_neg( ); + void set_immediate_operand_type( ); + void version_header(double a); + void maxnt_id(int x, int y, int z); + void parse_error_impl( const char *file, unsigned line, const char *msg, ... ); + void parse_assert_impl( int test_value, const char *file, unsigned line, const char *msg, ... ); + //Jin: handle instructino group for cdp + void start_inst_group(); + void end_inst_group(); + bool check_for_duplicates( const char *identifier ); + void read_parser_environment_variables(); + void set_ptx_warp_size(const struct core_config * warp_size); + const class ptx_instruction *ptx_instruction_lookup( const char *filename, unsigned linenumber ); - // member function list - void init_directive_state(); - void init_instruction_state(); - void start_function(int entry_point); - void add_function_name(const char *fname); - void add_directive(); - void end_function(); - void add_identifier(const char *s, int array_dim, unsigned array_ident); - void add_function_arg(); - void add_scalar_type_spec(int type_spec); - void add_scalar_operand(const char *identifier); - void add_neg_pred_operand(const char *identifier); - void add_variables(); - void set_variable_type(); - void add_opcode(int opcode); - void add_pred(const char *identifier, int negate, int predModifier); - void add_1vector_operand(const char *d1); - void add_2vector_operand(const char *d1, const char *d2); - void add_3vector_operand(const char *d1, const char *d2, const char *d3); - void add_4vector_operand(const char *d1, const char *d2, const char *d3, - const char *d4); - void add_8vector_operand(const char *d1, const char *d2, const char *d3, - const char *d4, const char *d5, const char *d6, - const char *d7, const char *d8); - void add_option(int option); - void add_wmma_option(int option); - void add_builtin_operand(int builtin, int dim_modifier); - void add_memory_operand(); - void add_literal_int(int value); - void add_literal_float(float value); - void add_literal_double(double value); - void add_address_operand(const char *identifier, int offset); - void add_address_operand2(int offset); - void add_label(const char *idenfiier); - void add_vector_spec(int spec); - void add_space_spec(enum _memory_space_t spec, int value); - void add_ptr_spec(enum _memory_space_t spec); - void add_extern_spec(); - void add_instruction(); - void set_return(); - void add_alignment_spec(int spec); - void add_array_initializer(); - void add_file(unsigned num, const char *filename); - void add_version_info(float ver, unsigned ext); - void *reset_symtab(); - void set_symtab(void *); - void add_pragma(const char *str); - void func_header(const char *a); - void func_header_info(const char *a); - void func_header_info_int(const char *a, int b); - void add_constptr(const char *identifier1, const char *identifier2, - int offset); - void target_header(char *a); - void target_header2(char *a, char *b); - void target_header3(char *a, char *b, char *c); - void add_double_operand(const char *d1, const char *d2); - void change_memory_addr_space(const char *identifier); - void change_operand_lohi(int lohi); - void change_double_operand_type(int addr_type); - void change_operand_neg(); - void set_immediate_operand_type(); - void version_header(double a); - void maxnt_id(int x, int y, int z); - void parse_error_impl(const char *file, unsigned line, const char *msg, ...); - void parse_assert_impl(int test_value, const char *file, unsigned line, - const char *msg, ...); - // Jin: handle instructino group for cdp - void start_inst_group(); - void end_inst_group(); - bool check_for_duplicates(const char *identifier); - void read_parser_environment_variables(); - void set_ptx_warp_size(const struct core_config *warp_size); - const class ptx_instruction *ptx_instruction_lookup(const char *filename, - unsigned linenumber); }; -const char *decode_token(int type); +const char *decode_token( int type ); void read_parser_environment_variables(); #define NON_ARRAY_IDENTIFIER 1 diff --git a/src/cuda-sim/ptx_sim.cc b/src/cuda-sim/ptx_sim.cc index 84bc3f3..949ee66 100644 --- a/src/cuda-sim/ptx_sim.cc +++ b/src/cuda-sim/ptx_sim.cc @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -31,588 +29,573 @@ #include <string> #include "ptx_ir.h" class ptx_recognizer; -typedef void *yyscan_t; -#include "../../libcuda/gpgpu_context.h" +typedef void * yyscan_t; +#include "ptx.tab.h" #include "../gpgpu-sim/gpu-sim.h" #include "../gpgpu-sim/shader.h" -#include "ptx.tab.h" +#include "../../libcuda/gpgpu_context.h" -void feature_not_implemented(const char *f); +void feature_not_implemented( const char *f ); -ptx_cta_info::ptx_cta_info(unsigned sm_idx, gpgpu_context *ctx) { - assert(ctx->func_sim->g_ptx_cta_info_sm_idx_used.find(sm_idx) == - ctx->func_sim->g_ptx_cta_info_sm_idx_used.end()); - ctx->func_sim->g_ptx_cta_info_sm_idx_used.insert(sm_idx); - m_sm_idx = sm_idx; - m_uid = (ctx->g_ptx_cta_info_uid)++; - m_bar_threads = 0; - gpgpu_ctx = ctx; +ptx_cta_info::ptx_cta_info( unsigned sm_idx, gpgpu_context* ctx ) +{ + assert( ctx->func_sim->g_ptx_cta_info_sm_idx_used.find(sm_idx) == ctx->func_sim->g_ptx_cta_info_sm_idx_used.end() ); + ctx->func_sim->g_ptx_cta_info_sm_idx_used.insert(sm_idx); + + m_sm_idx = sm_idx; + m_uid = (ctx->g_ptx_cta_info_uid)++; + m_bar_threads = 0; + gpgpu_ctx = ctx; } -void ptx_cta_info::add_thread(ptx_thread_info *thd) { - m_threads_in_cta.insert(thd); +void ptx_cta_info::add_thread( ptx_thread_info *thd ) +{ + m_threads_in_cta.insert(thd); } -unsigned ptx_cta_info::num_threads() const { return m_threads_in_cta.size(); } +unsigned ptx_cta_info::num_threads() const +{ + return m_threads_in_cta.size(); +} -void ptx_cta_info::check_cta_thread_status_and_reset() { - bool fail = false; - if (m_threads_that_have_exited.size() != m_threads_in_cta.size()) { - printf("\n\n"); - printf( - "Execution error: Some threads still running in CTA during CTA " - "reallocation! (1)\n"); - printf(" CTA uid = %Lu (sm_idx = %u) : %lu running out of %zu total\n", - m_uid, m_sm_idx, - (m_threads_in_cta.size() - m_threads_that_have_exited.size()), - m_threads_in_cta.size()); - printf(" These are the threads that are still running:\n"); - std::set<ptx_thread_info *>::iterator t_iter; - for (t_iter = m_threads_in_cta.begin(); t_iter != m_threads_in_cta.end(); - ++t_iter) { - ptx_thread_info *t = *t_iter; - if (m_threads_that_have_exited.find(t) == - m_threads_that_have_exited.end()) { - if (m_dangling_pointers.find(t) != m_dangling_pointers.end()) { - printf(" <thread deleted>\n"); - } else { - printf(" [done=%c] : ", (t->is_done() ? 'Y' : 'N')); - t->print_insn(t->get_pc(), stdout); - printf("\n"); - } +void ptx_cta_info::check_cta_thread_status_and_reset() +{ + bool fail = false; + if ( m_threads_that_have_exited.size() != m_threads_in_cta.size() ) { + printf("\n\n"); + printf("Execution error: Some threads still running in CTA during CTA reallocation! (1)\n"); + printf(" CTA uid = %Lu (sm_idx = %u) : %lu running out of %zu total\n", + m_uid, + m_sm_idx, + (m_threads_in_cta.size() - m_threads_that_have_exited.size()), m_threads_in_cta.size() ); + printf(" These are the threads that are still running:\n"); + std::set<ptx_thread_info*>::iterator t_iter; + for ( t_iter=m_threads_in_cta.begin(); t_iter != m_threads_in_cta.end(); ++t_iter ) { + ptx_thread_info *t = *t_iter; + if ( m_threads_that_have_exited.find(t) == m_threads_that_have_exited.end() ) { + if ( m_dangling_pointers.find(t) != m_dangling_pointers.end() ) { + printf(" <thread deleted>\n"); + } else { + printf(" [done=%c] : ", (t->is_done()?'Y':'N') ); + t->print_insn( t->get_pc(), stdout ); + printf("\n"); + } + } } - } - printf("\n\n"); - fail = true; - } - if (fail) { - abort(); - } + printf("\n\n"); + fail = true; + } + if ( fail ) { + abort(); + } - bool fail2 = false; - std::set<ptx_thread_info *>::iterator t_iter; - for (t_iter = m_threads_in_cta.begin(); t_iter != m_threads_in_cta.end(); - ++t_iter) { - ptx_thread_info *t = *t_iter; - if (m_dangling_pointers.find(t) == m_dangling_pointers.end()) { - if (!t->is_done()) { - if (!fail2) { - printf( - "Execution error: Some threads still running in CTA during CTA " - "reallocation! (2)\n"); - printf(" CTA uid = %Lu (sm_idx = %u) :\n", m_uid, m_sm_idx); - fail2 = true; - } - printf(" "); - t->print_insn(t->get_pc(), stdout); - printf("\n"); + bool fail2 = false; + std::set<ptx_thread_info*>::iterator t_iter; + for ( t_iter=m_threads_in_cta.begin(); t_iter != m_threads_in_cta.end(); ++t_iter ) { + ptx_thread_info *t = *t_iter; + if ( m_dangling_pointers.find(t) == m_dangling_pointers.end() ) { + if ( !t->is_done() ) { + if ( !fail2 ) { + printf("Execution error: Some threads still running in CTA during CTA reallocation! (2)\n"); + printf(" CTA uid = %Lu (sm_idx = %u) :\n", m_uid, m_sm_idx ); + fail2 = true; + } + printf(" "); + t->print_insn( t->get_pc(), stdout ); + printf("\n"); + } } - } - } - if (fail2) { - abort(); - } - m_threads_in_cta.clear(); - m_threads_that_have_exited.clear(); - m_dangling_pointers.clear(); + } + if ( fail2 ) { + abort(); + } + m_threads_in_cta.clear(); + m_threads_that_have_exited.clear(); + m_dangling_pointers.clear(); } -void ptx_cta_info::register_thread_exit(ptx_thread_info *thd) { - assert(m_threads_that_have_exited.find(thd) == - m_threads_that_have_exited.end()); - m_threads_that_have_exited.insert(thd); +void ptx_cta_info::register_thread_exit( ptx_thread_info *thd ) +{ + assert( m_threads_that_have_exited.find(thd) == m_threads_that_have_exited.end() ); + m_threads_that_have_exited.insert(thd); } -void ptx_cta_info::register_deleted_thread(ptx_thread_info *thd) { - m_dangling_pointers.insert(thd); +void ptx_cta_info::register_deleted_thread( ptx_thread_info *thd ) +{ + m_dangling_pointers.insert(thd); } -unsigned ptx_cta_info::get_sm_idx() const { return m_sm_idx; } +unsigned ptx_cta_info::get_sm_idx() const +{ + return m_sm_idx; +} -unsigned ptx_cta_info::get_bar_threads() const { return m_bar_threads; } +unsigned ptx_cta_info::get_bar_threads() const +{ + return m_bar_threads; +} -void ptx_cta_info::inc_bar_threads() { m_bar_threads++; } +void ptx_cta_info::inc_bar_threads() +{ + m_bar_threads++; +} -void ptx_cta_info::reset_bar_threads() { m_bar_threads = 0; } +void ptx_cta_info::reset_bar_threads() +{ + m_bar_threads = 0; +} + +ptx_warp_info::ptx_warp_info() +{ + reset_done_threads(); +} -ptx_warp_info::ptx_warp_info() { reset_done_threads(); } +unsigned ptx_warp_info::get_done_threads() const +{ + return m_done_threads; +} -unsigned ptx_warp_info::get_done_threads() const { return m_done_threads; } +void ptx_warp_info::inc_done_threads() +{ + m_done_threads++; +} -void ptx_warp_info::inc_done_threads() { m_done_threads++; } +void ptx_warp_info::reset_done_threads() +{ + m_done_threads = 0; +} -void ptx_warp_info::reset_done_threads() { m_done_threads = 0; } -ptx_thread_info::~ptx_thread_info() { - m_gpu->gpgpu_ctx->func_sim->g_ptx_thread_info_delete_count++; +ptx_thread_info::~ptx_thread_info() +{ + m_gpu->gpgpu_ctx->func_sim->g_ptx_thread_info_delete_count++; } -ptx_thread_info::ptx_thread_info(kernel_info_t &kernel) : m_kernel(kernel) { - m_uid = kernel.entry()->gpgpu_ctx->func_sim->g_ptx_thread_info_uid_next++; - m_core = NULL; - m_barrier_num = -1; - m_at_barrier = false; - m_valid = false; - m_gridid = 0; - m_thread_done = false; - m_cycle_done = 0; - m_PC = 0; - m_icount = 0; - m_last_effective_address = 0; - m_last_memory_space = undefined_space; - m_branch_taken = 0; - m_shared_mem = NULL; - m_sstarr_mem = NULL; - m_warp_info = NULL; - m_cta_info = NULL; - m_local_mem = NULL; - m_symbol_table = NULL; - m_func_info = NULL; - m_hw_tid = -1; - m_hw_wid = -1; - m_hw_sid = -1; - m_last_dram_callback.function = NULL; - m_last_dram_callback.instruction = NULL; - m_regs.push_back(reg_map_t()); - m_debug_trace_regs_modified.push_back(reg_map_t()); - m_debug_trace_regs_read.push_back(reg_map_t()); - m_callstack.push_back(stack_entry()); - m_RPC = -1; - m_RPC_updated = false; - m_last_was_call = false; - m_enable_debug_trace = false; - m_local_mem_stack_pointer = 0; - m_gpu = NULL; - m_last_set_operand_value = ptx_reg_t(); +ptx_thread_info::ptx_thread_info( kernel_info_t &kernel ) + : m_kernel(kernel) +{ + m_uid = kernel.entry()->gpgpu_ctx->func_sim->g_ptx_thread_info_uid_next++; + m_core = NULL; + m_barrier_num = -1; + m_at_barrier = false; + m_valid = false; + m_gridid = 0; + m_thread_done = false; + m_cycle_done = 0; + m_PC=0; + m_icount = 0; + m_last_effective_address = 0; + m_last_memory_space = undefined_space; + m_branch_taken = 0; + m_shared_mem = NULL; + m_sstarr_mem = NULL; + m_warp_info = NULL; + m_cta_info = NULL; + m_local_mem = NULL; + m_symbol_table = NULL; + m_func_info = NULL; + m_hw_tid = -1; + m_hw_wid = -1; + m_hw_sid = -1; + m_last_dram_callback.function = NULL; + m_last_dram_callback.instruction = NULL; + m_regs.push_back( reg_map_t() ); + m_debug_trace_regs_modified.push_back( reg_map_t() ); + m_debug_trace_regs_read.push_back( reg_map_t() ); + m_callstack.push_back( stack_entry() ); + m_RPC = -1; + m_RPC_updated = false; + m_last_was_call = false; + m_enable_debug_trace = false; + m_local_mem_stack_pointer = 0; + m_gpu = NULL; + m_last_set_operand_value=ptx_reg_t(); } -const ptx_version &ptx_thread_info::get_ptx_version() const { - return m_func_info->get_ptx_version(); +const ptx_version &ptx_thread_info::get_ptx_version() const +{ + return m_func_info->get_ptx_version(); } -void ptx_thread_info::set_done() { - assert(!m_at_barrier); - m_thread_done = true; - m_cycle_done = m_gpu->gpu_sim_cycle; +void ptx_thread_info::set_done() +{ + assert( !m_at_barrier ); + m_thread_done = true; + m_cycle_done = m_gpu->gpu_sim_cycle; } -unsigned ptx_thread_info::get_builtin(int builtin_id, unsigned dim_mod) { - assert(m_valid); - switch ((builtin_id & 0xFFFF)) { - case CLOCK_REG: +unsigned ptx_thread_info::get_builtin( int builtin_id, unsigned dim_mod ) +{ + assert( m_valid ); + switch ((builtin_id&0xFFFF)) { + case CLOCK_REG: return (unsigned)(m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); - case CLOCK64_REG: - abort(); // change return value to unsigned long long? - // GPGPUSim clock is 4 times slower - multiply by 4 - return (m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle) * 4; - case HALFCLOCK_ID: + case CLOCK64_REG: + abort(); // change return value to unsigned long long? + // GPGPUSim clock is 4 times slower - multiply by 4 + return (m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle)*4; + case HALFCLOCK_ID: // GPGPUSim clock is 4 times slower - multiply by 4 - // Hardware clock counter is incremented at half the shader clock - // frequency - divide by 2 (Henry '10) - return (m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle) * 2; - case CTAID_REG: - assert(dim_mod < 3); - if (dim_mod == 0) return m_ctaid.x; - if (dim_mod == 1) return m_ctaid.y; - if (dim_mod == 2) return m_ctaid.z; + // Hardware clock counter is incremented at half the shader clock frequency - divide by 2 (Henry '10) + return (m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle)*2; + case CTAID_REG: + assert( dim_mod < 3 ); + if( dim_mod == 0 ) return m_ctaid.x; + if( dim_mod == 1 ) return m_ctaid.y; + if( dim_mod == 2 ) return m_ctaid.z; abort(); break; - case ENVREG_REG: { - int index = builtin_id >> 16; - dim3 gdim = this->get_core()->get_kernel_info()->get_grid_dim(); - switch (index) { - case 0: - case 1: - case 2: - case 3: - case 4: - case 5: - return 0; - break; - case 6: - return gdim.x; - case 7: - return gdim.y; - case 8: - return gdim.z; - case 9: - if (gdim.z == 1 && gdim.y == 1) - return 1; - else if (gdim.z == 1) - return 2; - else - return 3; - break; - default: - break; - } - } - case GRIDID_REG: + case ENVREG_REG:{ + int index = builtin_id >> 16; + dim3 gdim = this->get_core()->get_kernel_info()->get_grid_dim(); + switch(index){ + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + return 0; + break; + case 6: + return gdim.x; + case 7: + return gdim.y; + case 8: + return gdim.z; + case 9: + if(gdim.z == 1 && gdim.y == 1) + return 1; + else if(gdim.z == 1) + return 2; + else + return 3; + break; + default: + break; + } + } + case GRIDID_REG: return m_gridid; - case LANEID_REG: - return get_hw_tid() % m_core->get_warp_size(); - case LANEMASK_EQ_REG: - feature_not_implemented("%lanemask_eq"); - return 0; - case LANEMASK_LE_REG: - feature_not_implemented("%lanemask_le"); - return 0; - case LANEMASK_LT_REG: - feature_not_implemented("%lanemask_lt"); - return 0; - case LANEMASK_GE_REG: - feature_not_implemented("%lanemask_ge"); - return 0; - case LANEMASK_GT_REG: - feature_not_implemented("%lanemask_gt"); - return 0; - case NCTAID_REG: - assert(dim_mod < 3); - if (dim_mod == 0) return m_nctaid.x; - if (dim_mod == 1) return m_nctaid.y; - if (dim_mod == 2) return m_nctaid.z; + case LANEID_REG: return get_hw_tid() % m_core->get_warp_size(); + case LANEMASK_EQ_REG: feature_not_implemented( "%lanemask_eq" ); return 0; + case LANEMASK_LE_REG: feature_not_implemented( "%lanemask_le" ); return 0; + case LANEMASK_LT_REG: feature_not_implemented( "%lanemask_lt" ); return 0; + case LANEMASK_GE_REG: feature_not_implemented( "%lanemask_ge" ); return 0; + case LANEMASK_GT_REG: feature_not_implemented( "%lanemask_gt" ); return 0; + case NCTAID_REG: + assert( dim_mod < 3 ); + if( dim_mod == 0 ) return m_nctaid.x; + if( dim_mod == 1 ) return m_nctaid.y; + if( dim_mod == 2 ) return m_nctaid.z; abort(); break; - case NTID_REG: - assert(dim_mod < 3); - if (dim_mod == 0) return m_ntid.x; - if (dim_mod == 1) return m_ntid.y; - if (dim_mod == 2) return m_ntid.z; + case NTID_REG: + assert( dim_mod < 3 ); + if( dim_mod == 0 ) return m_ntid.x; + if( dim_mod == 1 ) return m_ntid.y; + if( dim_mod == 2 ) return m_ntid.z; abort(); break; - case NWARPID_REG: - feature_not_implemented("%nwarpid"); - return 0; - case PM_REG: - feature_not_implemented("%pm"); - return 0; - case SMID_REG: - feature_not_implemented("%smid"); - return 0; - case TID_REG: - assert(dim_mod < 3); - if (dim_mod == 0) return m_tid.x; - if (dim_mod == 1) return m_tid.y; - if (dim_mod == 2) return m_tid.z; + case NWARPID_REG: feature_not_implemented( "%nwarpid" ); return 0; + case PM_REG: feature_not_implemented( "%pm" ); return 0; + case SMID_REG: feature_not_implemented( "%smid" ); return 0; + case TID_REG: + assert( dim_mod < 3 ); + if( dim_mod == 0 ) return m_tid.x; + if( dim_mod == 1 ) return m_tid.y; + if( dim_mod == 2 ) return m_tid.z; abort(); break; - case WARPSZ_REG: - return m_core->get_warp_size(); - default: + case WARPSZ_REG: return m_core->get_warp_size() ; + default: assert(0); - } - return 0; + } + return 0; } -void ptx_thread_info::set_info(function_info *func) { +void ptx_thread_info::set_info( function_info *func ) +{ m_symbol_table = func->get_symtab(); m_func_info = func; m_PC = func->get_start_PC(); } -void ptx_thread_info::cpy_tid_to_reg(dim3 tid) { - // copies %tid.x, %tid.y and %tid.z into $r0 - ptx_reg_t data; - data.s64 = 0; +void ptx_thread_info::cpy_tid_to_reg( dim3 tid ) +{ + //copies %tid.x, %tid.y and %tid.z into $r0 + ptx_reg_t data; + data.s64=0; - data.u32 = (tid.x + (tid.y << 16) + (tid.z << 26)); + data.u32=(tid.x + (tid.y<<16) + (tid.z<<26)); - const symbol *r0 = m_symbol_table->lookup("$r0"); - if (r0) { - // No need to set pid if kernel doesn't use it - set_reg(r0, data); - } + const symbol *r0 = m_symbol_table->lookup("$r0"); + if (r0){ + //No need to set pid if kernel doesn't use it + set_reg(r0,data); + } } -void ptx_thread_info::print_insn(unsigned pc, FILE *fp) const { - m_func_info->print_insn(pc, fp); +void ptx_thread_info::print_insn( unsigned pc, FILE * fp ) const +{ + m_func_info->print_insn(pc,fp); } -static void print_reg(FILE *fp, std::string name, ptx_reg_t value, - symbol_table *symtab) { - const symbol *sym = symtab->lookup(name.c_str()); - fprintf(fp, " %8s ", name.c_str()); - if (sym == NULL) { - fprintf(fp, "<unknown type> 0x%llx\n", (unsigned long long)value.u64); - return; - } - const type_info *t = sym->type(); - if (t == NULL) { - fprintf(fp, "<unknown type> 0x%llx\n", (unsigned long long)value.u64); - return; - } - type_info_key ti = t->get_key(); +static void print_reg( FILE *fp, std::string name, ptx_reg_t value, symbol_table *symtab ) +{ + const symbol *sym = symtab->lookup(name.c_str()); + fprintf(fp," %8s ", name.c_str() ); + if( sym == NULL ) { + fprintf(fp,"<unknown type> 0x%llx\n", (unsigned long long ) value.u64 ); + return; + } + const type_info *t = sym->type(); + if( t == NULL ) { + fprintf(fp,"<unknown type> 0x%llx\n", (unsigned long long ) value.u64 ); + return; + } + type_info_key ti = t->get_key(); - switch (ti.scalar_type()) { - case S8_TYPE: - fprintf(fp, ".s8 %d\n", value.s8); - break; - case S16_TYPE: - fprintf(fp, ".s16 %d\n", value.s16); - break; - case S32_TYPE: - fprintf(fp, ".s32 %d\n", value.s32); - break; - case S64_TYPE: - fprintf(fp, ".s64 %Ld\n", value.s64); - break; - case U8_TYPE: - fprintf(fp, ".u8 %u [0x%02x]\n", value.u8, (unsigned)value.u8); - break; - case U16_TYPE: - fprintf(fp, ".u16 %u [0x%04x]\n", value.u16, (unsigned)value.u16); + switch ( ti.scalar_type() ) { + case S8_TYPE: fprintf(fp,".s8 %d\n", value.s8 ); break; + case S16_TYPE: fprintf(fp,".s16 %d\n", value.s16 ); break; + case S32_TYPE: fprintf(fp,".s32 %d\n", value.s32 ); break; + case S64_TYPE: fprintf(fp,".s64 %Ld\n", value.s64 ); break; + case U8_TYPE: fprintf(fp,".u8 %u [0x%02x]\n", value.u8, (unsigned) value.u8 ); break; + case U16_TYPE: fprintf(fp,".u16 %u [0x%04x]\n", value.u16, (unsigned) value.u16 ); break; + case U32_TYPE: fprintf(fp,".u32 %u [0x%08x]\n", value.u32, (unsigned) value.u32 ); break; + case U64_TYPE: fprintf(fp,".u64 %llu [0x%llx]\n", value.u64, value.u64 ); break; + case F16_TYPE: fprintf(fp,".f16 %f [0x%04x]\n", value.f16, (unsigned) value.u16 ); break; + case F32_TYPE: fprintf(fp,".f32 %.15lf [0x%08x]\n", value.f32, value.u32 ); break; + case F64_TYPE: fprintf(fp,".f64 %.15le [0x%016llx]\n", value.f64, value.u64 ); break; + case B8_TYPE: fprintf(fp,".b8 0x%02x\n", (unsigned) value.u8 ); break; + case B16_TYPE: fprintf(fp,".b16 0x%04x\n", (unsigned) value.u16 ); break; + case B32_TYPE: fprintf(fp,".b32 0x%08x\n", (unsigned) value.u32 ); break; + case B64_TYPE: fprintf(fp,".b64 0x%llx\n", (unsigned long long ) value.u64 ); break; + case PRED_TYPE: fprintf(fp,".pred %u\n", (unsigned) value.pred ); break; + default: + fprintf( fp, "non-scalar type\n" ); break; - case U32_TYPE: - fprintf(fp, ".u32 %u [0x%08x]\n", value.u32, (unsigned)value.u32); - break; - case U64_TYPE: - fprintf(fp, ".u64 %llu [0x%llx]\n", value.u64, value.u64); - break; - case F16_TYPE: - fprintf(fp, ".f16 %f [0x%04x]\n", value.f16, (unsigned)value.u16); - break; - case F32_TYPE: - fprintf(fp, ".f32 %.15lf [0x%08x]\n", value.f32, value.u32); - break; - case F64_TYPE: - fprintf(fp, ".f64 %.15le [0x%016llx]\n", value.f64, value.u64); - break; - case B8_TYPE: - fprintf(fp, ".b8 0x%02x\n", (unsigned)value.u8); - break; - case B16_TYPE: - fprintf(fp, ".b16 0x%04x\n", (unsigned)value.u16); - break; - case B32_TYPE: - fprintf(fp, ".b32 0x%08x\n", (unsigned)value.u32); - break; - case B64_TYPE: - fprintf(fp, ".b64 0x%llx\n", (unsigned long long)value.u64); - break; - case PRED_TYPE: - fprintf(fp, ".pred %u\n", (unsigned)value.pred); - break; - default: - fprintf(fp, "non-scalar type\n"); - break; - } - fflush(fp); + } + fflush(fp); } -static void print_reg(std::string name, ptx_reg_t value, symbol_table *symtab) { - print_reg(stdout, name, value, symtab); +static void print_reg( std::string name, ptx_reg_t value, symbol_table *symtab ) +{ + print_reg(stdout,name,value,symtab); } -void ptx_thread_info::callstack_push(unsigned pc, unsigned rpc, - const symbol *return_var_src, - const symbol *return_var_dst, - unsigned call_uid) { - m_RPC = -1; - m_RPC_updated = true; - m_last_was_call = true; - assert(m_func_info != NULL); - m_callstack.push_back(stack_entry(m_symbol_table, m_func_info, pc, rpc, - return_var_src, return_var_dst, call_uid)); - m_regs.push_back(reg_map_t()); - m_debug_trace_regs_modified.push_back(reg_map_t()); - m_debug_trace_regs_read.push_back(reg_map_t()); - m_local_mem_stack_pointer += m_func_info->local_mem_framesize(); +void ptx_thread_info::callstack_push( unsigned pc, unsigned rpc, const symbol *return_var_src, const symbol *return_var_dst, unsigned call_uid ) +{ + m_RPC = -1; + m_RPC_updated = true; + m_last_was_call = true; + assert( m_func_info != NULL ); + m_callstack.push_back( stack_entry(m_symbol_table,m_func_info,pc,rpc,return_var_src,return_var_dst,call_uid) ); + m_regs.push_back( reg_map_t() ); + m_debug_trace_regs_modified.push_back( reg_map_t() ); + m_debug_trace_regs_read.push_back( reg_map_t() ); + m_local_mem_stack_pointer += m_func_info->local_mem_framesize(); } -// ptxplus version of callstack_push. -void ptx_thread_info::callstack_push_plus(unsigned pc, unsigned rpc, - const symbol *return_var_src, - const symbol *return_var_dst, - unsigned call_uid) { - m_RPC = -1; - m_RPC_updated = true; - m_last_was_call = true; - assert(m_func_info != NULL); - m_callstack.push_back(stack_entry(m_symbol_table, m_func_info, pc, rpc, - return_var_src, return_var_dst, call_uid)); - // m_regs.push_back( reg_map_t() ); - // m_debug_trace_regs_modified.push_back( reg_map_t() ); - // m_debug_trace_regs_read.push_back( reg_map_t() ); - m_local_mem_stack_pointer += m_func_info->local_mem_framesize(); +//ptxplus version of callstack_push. +void ptx_thread_info::callstack_push_plus( unsigned pc, unsigned rpc, const symbol *return_var_src, const symbol *return_var_dst, unsigned call_uid ) +{ + m_RPC = -1; + m_RPC_updated = true; + m_last_was_call = true; + assert( m_func_info != NULL ); + m_callstack.push_back( stack_entry(m_symbol_table,m_func_info,pc,rpc,return_var_src,return_var_dst,call_uid) ); + //m_regs.push_back( reg_map_t() ); + //m_debug_trace_regs_modified.push_back( reg_map_t() ); + //m_debug_trace_regs_read.push_back( reg_map_t() ); + m_local_mem_stack_pointer += m_func_info->local_mem_framesize(); } -bool ptx_thread_info::callstack_pop() { - const symbol *rv_src = m_callstack.back().m_return_var_src; - const symbol *rv_dst = m_callstack.back().m_return_var_dst; - assert(!((rv_src != NULL) ^ (rv_dst != NULL))); // ensure caller and callee - // agree on whether there is - // a return value - // read return value from callee frame - arg_buffer_t buffer(m_gpu->gpgpu_ctx); - if (rv_src != NULL) - buffer = copy_arg_to_buffer(this, operand_info(rv_src, m_gpu->gpgpu_ctx), - rv_dst); +bool ptx_thread_info::callstack_pop() +{ + const symbol *rv_src = m_callstack.back().m_return_var_src; + const symbol *rv_dst = m_callstack.back().m_return_var_dst; + assert( !((rv_src != NULL) ^ (rv_dst != NULL)) ); // ensure caller and callee agree on whether there is a return value + + // read return value from callee frame + arg_buffer_t buffer(m_gpu->gpgpu_ctx); + if( rv_src != NULL ) + buffer = copy_arg_to_buffer(this, operand_info(rv_src, m_gpu->gpgpu_ctx), rv_dst ); - m_symbol_table = m_callstack.back().m_symbol_table; - m_NPC = m_callstack.back().m_PC; - m_RPC_updated = true; - m_last_was_call = false; - m_RPC = m_callstack.back().m_RPC; - m_func_info = m_callstack.back().m_func_info; - if (m_func_info) { - assert(m_local_mem_stack_pointer >= m_func_info->local_mem_framesize()); - m_local_mem_stack_pointer -= m_func_info->local_mem_framesize(); - } - m_callstack.pop_back(); - m_regs.pop_back(); - m_debug_trace_regs_modified.pop_back(); - m_debug_trace_regs_read.pop_back(); + m_symbol_table = m_callstack.back().m_symbol_table; + m_NPC = m_callstack.back().m_PC; + m_RPC_updated = true; + m_last_was_call = false; + m_RPC = m_callstack.back().m_RPC; + m_func_info = m_callstack.back().m_func_info; + if( m_func_info ) { + assert( m_local_mem_stack_pointer >= m_func_info->local_mem_framesize() ); + m_local_mem_stack_pointer -= m_func_info->local_mem_framesize(); + } + m_callstack.pop_back(); + m_regs.pop_back(); + m_debug_trace_regs_modified.pop_back(); + m_debug_trace_regs_read.pop_back(); - // write return value into caller frame - if (rv_dst != NULL) copy_buffer_to_frame(this, buffer); + // write return value into caller frame + if( rv_dst != NULL ) + copy_buffer_to_frame(this, buffer); - return m_callstack.empty(); + return m_callstack.empty(); } -// ptxplus version of callstack_pop -bool ptx_thread_info::callstack_pop_plus() { - const symbol *rv_src = m_callstack.back().m_return_var_src; - const symbol *rv_dst = m_callstack.back().m_return_var_dst; - assert(!((rv_src != NULL) ^ (rv_dst != NULL))); // ensure caller and callee - // agree on whether there is - // a return value +//ptxplus version of callstack_pop +bool ptx_thread_info::callstack_pop_plus() +{ + const symbol *rv_src = m_callstack.back().m_return_var_src; + const symbol *rv_dst = m_callstack.back().m_return_var_dst; + assert( !((rv_src != NULL) ^ (rv_dst != NULL)) ); // ensure caller and callee agree on whether there is a return value - // read return value from callee frame - arg_buffer_t buffer(m_gpu->gpgpu_ctx); - if (rv_src != NULL) - buffer = copy_arg_to_buffer(this, operand_info(rv_src, m_gpu->gpgpu_ctx), - rv_dst); + // read return value from callee frame + arg_buffer_t buffer(m_gpu->gpgpu_ctx); + if( rv_src != NULL ) + buffer = copy_arg_to_buffer(this, operand_info(rv_src, m_gpu->gpgpu_ctx), rv_dst ); - m_symbol_table = m_callstack.back().m_symbol_table; - m_NPC = m_callstack.back().m_PC; - m_RPC_updated = true; - m_last_was_call = false; - m_RPC = m_callstack.back().m_RPC; - m_func_info = m_callstack.back().m_func_info; - if (m_func_info) { - assert(m_local_mem_stack_pointer >= m_func_info->local_mem_framesize()); - m_local_mem_stack_pointer -= m_func_info->local_mem_framesize(); - } - m_callstack.pop_back(); - // m_regs.pop_back(); - // m_debug_trace_regs_modified.pop_back(); - // m_debug_trace_regs_read.pop_back(); + m_symbol_table = m_callstack.back().m_symbol_table; + m_NPC = m_callstack.back().m_PC; + m_RPC_updated = true; + m_last_was_call = false; + m_RPC = m_callstack.back().m_RPC; + m_func_info = m_callstack.back().m_func_info; + if( m_func_info ) { + assert( m_local_mem_stack_pointer >= m_func_info->local_mem_framesize() ); + m_local_mem_stack_pointer -= m_func_info->local_mem_framesize(); + } + m_callstack.pop_back(); + //m_regs.pop_back(); + //m_debug_trace_regs_modified.pop_back(); + //m_debug_trace_regs_read.pop_back(); - // write return value into caller frame - if (rv_dst != NULL) copy_buffer_to_frame(this, buffer); + // write return value into caller frame + if( rv_dst != NULL ) + copy_buffer_to_frame(this, buffer); - return m_callstack.empty(); + return m_callstack.empty(); } -void ptx_thread_info::dump_callstack() const { - std::list<stack_entry>::const_iterator c = m_callstack.begin(); - std::list<reg_map_t>::const_iterator r = m_regs.begin(); +void ptx_thread_info::dump_callstack() const +{ + std::list<stack_entry>::const_iterator c=m_callstack.begin(); + std::list<reg_map_t>::const_iterator r=m_regs.begin(); - printf("\n\n"); - printf("Call stack for thread uid = %u (sc=%u, hwtid=%u)\n", m_uid, m_hw_sid, - m_hw_tid); - while (c != m_callstack.end() && r != m_regs.end()) { - const stack_entry &c_e = *c; - const reg_map_t ®s = *r; - if (!c_e.m_valid) { - printf(" <entry> #regs = %zu\n", - regs.size()); - } else { - printf(" %20s PC=%3u RV= (callee=\'%s\',caller=\'%s\') #regs = %zu\n", - c_e.m_func_info->get_name().c_str(), c_e.m_PC, - c_e.m_return_var_src->name().c_str(), - c_e.m_return_var_dst->name().c_str(), regs.size()); - } - c++; - r++; - } - if (c != m_callstack.end() || r != m_regs.end()) { - printf(" *** mismatch in m_regs and m_callstack sizes ***\n"); - } - printf("\n\n"); -} - -std::string ptx_thread_info::get_location() const { - const ptx_instruction *pI = m_func_info->get_instruction(m_PC); - char buf[1024]; - snprintf(buf, 1024, "%s:%u", pI->source_file(), pI->source_line()); - return std::string(buf); + printf("\n\n"); + printf("Call stack for thread uid = %u (sc=%u, hwtid=%u)\n", m_uid, m_hw_sid, m_hw_tid ); + while( c != m_callstack.end() && r != m_regs.end() ) { + const stack_entry &c_e = *c; + const reg_map_t ®s = *r; + if( !c_e.m_valid ) { + printf(" <entry> #regs = %zu\n", regs.size() ); + } else { + printf(" %20s PC=%3u RV= (callee=\'%s\',caller=\'%s\') #regs = %zu\n", + c_e.m_func_info->get_name().c_str(), c_e.m_PC, + c_e.m_return_var_src->name().c_str(), + c_e.m_return_var_dst->name().c_str(), + regs.size() ); + } + c++; + r++; + } + if( c != m_callstack.end() || r != m_regs.end() ) { + printf(" *** mismatch in m_regs and m_callstack sizes ***\n" ); + } + printf("\n\n"); } -const ptx_instruction *ptx_thread_info::get_inst() const { - return m_func_info->get_instruction(m_PC); +std::string ptx_thread_info::get_location() const +{ + const ptx_instruction *pI = m_func_info->get_instruction(m_PC); + char buf[1024]; + snprintf(buf,1024,"%s:%u", pI->source_file(), pI->source_line() ); + return std::string(buf); } -const ptx_instruction *ptx_thread_info::get_inst(addr_t pc) const { - return m_func_info->get_instruction(pc); +const ptx_instruction *ptx_thread_info::get_inst() const +{ + return m_func_info->get_instruction(m_PC); } -void ptx_thread_info::dump_regs(FILE *fp) { - if (m_regs.empty()) return; - if (m_regs.back().empty()) return; - fprintf(fp, "Register File Contents:\n"); - fflush(fp); - reg_map_t::const_iterator r; - for (r = m_regs.back().begin(); r != m_regs.back().end(); ++r) { - const symbol *sym = r->first; - ptx_reg_t value = r->second; - std::string name = sym->name(); - print_reg(fp, name, value, m_symbol_table); - } +const ptx_instruction *ptx_thread_info::get_inst( addr_t pc ) const +{ + return m_func_info->get_instruction(pc); } -void ptx_thread_info::dump_modifiedregs(FILE *fp) { - if (!(m_debug_trace_regs_modified.empty() || - m_debug_trace_regs_modified.back().empty())) { - fprintf(fp, "Output Registers:\n"); - fflush(fp); - reg_map_t::iterator r; - for (r = m_debug_trace_regs_modified.back().begin(); - r != m_debug_trace_regs_modified.back().end(); ++r) { +void ptx_thread_info::dump_regs( FILE *fp ) +{ + if(m_regs.empty()) return; + if(m_regs.back().empty()) return; + fprintf(fp,"Register File Contents:\n"); + fflush(fp); + reg_map_t::const_iterator r; + for ( r=m_regs.back().begin(); r != m_regs.back().end(); ++r ) { const symbol *sym = r->first; - std::string name = sym->name(); ptx_reg_t value = r->second; - print_reg(fp, name, value, m_symbol_table); - } - } - if (!(m_debug_trace_regs_read.empty() || - m_debug_trace_regs_read.back().empty())) { - fprintf(fp, "Input Registers:\n"); - fflush(fp); - reg_map_t::iterator r; - for (r = m_debug_trace_regs_read.back().begin(); - r != m_debug_trace_regs_read.back().end(); ++r) { - const symbol *sym = r->first; std::string name = sym->name(); - ptx_reg_t value = r->second; - print_reg(fp, name, value, m_symbol_table); - } - } + print_reg(fp,name,value,m_symbol_table); + } } -void ptx_thread_info::push_breakaddr(const operand_info &breakaddr) { - m_breakaddrs.push(breakaddr); +void ptx_thread_info::dump_modifiedregs(FILE *fp) +{ + if( !(m_debug_trace_regs_modified.empty() || + m_debug_trace_regs_modified.back().empty()) ) { + fprintf(fp,"Output Registers:\n"); + fflush(fp); + reg_map_t::iterator r; + for ( r=m_debug_trace_regs_modified.back().begin(); r != m_debug_trace_regs_modified.back().end(); ++r ) { + const symbol *sym = r->first; + std::string name = sym->name(); + ptx_reg_t value = r->second; + print_reg(fp,name,value,m_symbol_table); + } + } + if( !(m_debug_trace_regs_read.empty() || + m_debug_trace_regs_read.back().empty()) ) { + fprintf(fp,"Input Registers:\n"); + fflush(fp); + reg_map_t::iterator r; + for ( r=m_debug_trace_regs_read.back().begin(); r != m_debug_trace_regs_read.back().end(); ++r ) { + const symbol *sym = r->first; + std::string name = sym->name(); + ptx_reg_t value = r->second; + print_reg(fp,name,value,m_symbol_table); + } + } +} + +void ptx_thread_info::push_breakaddr(const operand_info &breakaddr) +{ + m_breakaddrs.push(breakaddr); } -const operand_info &ptx_thread_info::pop_breakaddr() { - if (m_breakaddrs.empty()) { - printf("empty breakaddrs stack"); - assert(0); - } - operand_info &breakaddr = m_breakaddrs.top(); - m_breakaddrs.pop(); - return breakaddr; +const operand_info& ptx_thread_info::pop_breakaddr() +{ + if(m_breakaddrs.empty()) { + printf("empty breakaddrs stack"); + assert(0); + } + operand_info& breakaddr = m_breakaddrs.top(); + m_breakaddrs.pop(); + return breakaddr; } -void ptx_thread_info::set_npc(const function_info *f) { - m_NPC = f->get_start_PC(); - m_func_info = const_cast<function_info *>(f); - m_symbol_table = m_func_info->get_symtab(); +void ptx_thread_info::set_npc( const function_info *f ) +{ + m_NPC = f->get_start_PC(); + m_func_info = const_cast<function_info*>( f ); + m_symbol_table = m_func_info->get_symtab(); } -void feature_not_implemented(const char *f) { - printf("GPGPU-Sim: feature '%s' not supported\n", f); - abort(); + +void feature_not_implemented( const char *f ) +{ + printf("GPGPU-Sim: feature '%s' not supported\n", f ); + abort(); } diff --git a/src/cuda-sim/ptx_sim.h b/src/cuda-sim/ptx_sim.h index 372bcfd..c2b3cc8 100644 --- a/src/cuda-sim/ptx_sim.h +++ b/src/cuda-sim/ptx_sim.h @@ -6,16 +6,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -30,28 +28,31 @@ #define ptx_sim_h_INCLUDED #include <stdlib.h> -#include "../abstract_hardware_model.h" -#include "../tr1_hash_map.h" #include "half.h" +#include "../abstract_hardware_model.h" +#include "../tr1_hash_map.h" #include <assert.h> #include "opcodes.h" -#include <list> +#include <string> #include <map> #include <set> -#include <string> +#include <list> #include "memory.h" -#define GCC_VERSION \ - (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) +#define GCC_VERSION (__GNUC__ * 10000 \ + + __GNUC_MINOR__ * 100 \ + + __GNUC_PATCHLEVEL__) + + struct param_t { - const void *pdata; - int type; - size_t size; - size_t offset; + const void *pdata; + int type; + size_t size; + size_t offset; }; #include <stack> @@ -61,93 +62,98 @@ struct param_t { using half_float::half; union ptx_reg_t { - ptx_reg_t() { - bits.ms = 0; - bits.ls = 0; - u128.low = 0; - u128.lowest = 0; - u128.highest = 0; - u128.high = 0; - s8 = 0; - s16 = 0; - s32 = 0; - s64 = 0; - u8 = 0; - u16 = 0; - u64 = 0; - f16 = 0; - f32 = 0; - f64 = 0; - pred = 0; - } - ptx_reg_t(unsigned x) { - bits.ms = 0; - bits.ls = 0; - u128.low = 0; - u128.lowest = 0; - u128.highest = 0; - u128.high = 0; - s8 = 0; - s16 = 0; - s32 = 0; - s64 = 0; - u8 = 0; - u16 = 0; - u64 = 0; - f16 = 0; - f32 = 0; - f64 = 0; - pred = 0; - u32 = x; - } - operator unsigned int() { return u32; } - operator unsigned short() { return u16; } - operator unsigned char() { return u8; } - operator unsigned long long() { return u64; } + ptx_reg_t() { + bits.ms = 0; + bits.ls = 0; + u128.low=0; + u128.lowest=0; + u128.highest=0; + u128.high=0; + s8=0; + s16=0; + s32=0; + s64=0; + u8=0; + u16=0; + u64=0; + f16=0; + f32=0; + f64=0; + pred=0; + } + ptx_reg_t(unsigned x) + { + bits.ms = 0; + bits.ls = 0; + u128.low=0; + u128.lowest=0; + u128.highest=0; + u128.high=0; + s8=0; + s16=0; + s32=0; + s64=0; + u8=0; + u16=0; + u64=0; + f16=0; + f32=0; + f64=0; + pred=0; + u32 = x; + } + operator unsigned int() { return u32;} + operator unsigned short() { return u16;} + operator unsigned char() { return u8;} + operator unsigned long long() { return u64;} - void mask_and(unsigned ms, unsigned ls) { - bits.ms &= ms; - bits.ls &= ls; - } + void mask_and( unsigned ms, unsigned ls ) + { + bits.ms &= ms; + bits.ls &= ls; + } - void mask_or(unsigned ms, unsigned ls) { - bits.ms |= ms; - bits.ls |= ls; - } - int get_bit(unsigned bit) { - if (bit < 32) - return (bits.ls >> bit) & 1; - else - return (bits.ms >> (bit - 32)) & 1; - } + void mask_or( unsigned ms, unsigned ls ) + { + bits.ms |= ms; + bits.ls |= ls; + } + int get_bit( unsigned bit ) + { + if ( bit < 32 ) + return(bits.ls >> bit) & 1; + else + return(bits.ms >> (bit-32)) & 1; + } + + signed char s8; + signed short s16; + signed int s32; + signed long long s64; + unsigned char u8; + unsigned short u16; + unsigned int u32; + unsigned long long u64; + //gcc 4.7.0 + #if GCC_VERSION >= 40700 + half f16; + #else + float f16; + #endif + float f32; + double f64; + struct { + unsigned ls; + unsigned ms; + } bits; + struct { + unsigned int lowest; + unsigned int low; + unsigned int high; + unsigned int highest; + } u128; + unsigned pred : 4; - signed char s8; - signed short s16; - signed int s32; - signed long long s64; - unsigned char u8; - unsigned short u16; - unsigned int u32; - unsigned long long u64; -// gcc 4.7.0 -#if GCC_VERSION >= 40700 - half f16; -#else - float f16; -#endif - float f32; - double f64; - struct { - unsigned ls; - unsigned ms; - } bits; - struct { - unsigned int lowest; - unsigned int low; - unsigned int high; - unsigned int highest; - } u128; - unsigned pred : 4; }; class ptx_instruction; @@ -157,375 +163,373 @@ class function_info; class ptx_thread_info; class ptx_cta_info { - public: - ptx_cta_info(unsigned sm_idx, gpgpu_context *ctx); - void add_thread(ptx_thread_info *thd); - unsigned num_threads() const; - void check_cta_thread_status_and_reset(); - void register_thread_exit(ptx_thread_info *thd); - void register_deleted_thread(ptx_thread_info *thd); - unsigned get_sm_idx() const; - unsigned get_bar_threads() const; - void inc_bar_threads(); - void reset_bar_threads(); +public: + ptx_cta_info( unsigned sm_idx, gpgpu_context* ctx ); + void add_thread( ptx_thread_info *thd ); + unsigned num_threads() const; + void check_cta_thread_status_and_reset(); + void register_thread_exit( ptx_thread_info *thd ); + void register_deleted_thread( ptx_thread_info *thd ); + unsigned get_sm_idx() const; + unsigned get_bar_threads() const; + void inc_bar_threads(); + void reset_bar_threads(); - private: - // backward pointer - class gpgpu_context *gpgpu_ctx; - unsigned m_bar_threads; - unsigned long long m_uid; - unsigned m_sm_idx; - std::set<ptx_thread_info *> m_threads_in_cta; - std::set<ptx_thread_info *> m_threads_that_have_exited; - std::set<ptx_thread_info *> m_dangling_pointers; +private: + // backward pointer + class gpgpu_context* gpgpu_ctx; + unsigned m_bar_threads; + unsigned long long m_uid; + unsigned m_sm_idx; + std::set<ptx_thread_info*> m_threads_in_cta; + std::set<ptx_thread_info*> m_threads_that_have_exited; + std::set<ptx_thread_info*> m_dangling_pointers; }; class ptx_warp_info { - public: - ptx_warp_info(); // add get_core or something, or threads? - unsigned get_done_threads() const; - void inc_done_threads(); - void reset_done_threads(); +public: + ptx_warp_info(); // add get_core or something, or threads? + unsigned get_done_threads() const; + void inc_done_threads(); + void reset_done_threads(); - private: - unsigned m_done_threads; +private: + unsigned m_done_threads; }; class symbol; struct stack_entry { - stack_entry() { - m_symbol_table = NULL; - m_func_info = NULL; - m_PC = 0; - m_RPC = -1; - m_return_var_src = NULL; - m_return_var_dst = NULL; - m_call_uid = 0; - m_valid = false; - } - stack_entry(symbol_table *s, function_info *f, unsigned pc, unsigned rpc, - const symbol *return_var_src, const symbol *return_var_dst, - unsigned call_uid) { - m_symbol_table = s; - m_func_info = f; - m_PC = pc; - m_RPC = rpc; - m_return_var_src = return_var_src; - m_return_var_dst = return_var_dst; - m_call_uid = call_uid; - m_valid = true; - } + stack_entry() { + m_symbol_table=NULL; + m_func_info=NULL; + m_PC=0; + m_RPC=-1; + m_return_var_src = NULL; + m_return_var_dst = NULL; + m_call_uid = 0; + m_valid = false; + } + stack_entry( symbol_table *s, function_info *f, unsigned pc, unsigned rpc, const symbol *return_var_src, const symbol *return_var_dst, unsigned call_uid ) + { + m_symbol_table=s; + m_func_info=f; + m_PC=pc; + m_RPC=rpc; + m_return_var_src = return_var_src; + m_return_var_dst = return_var_dst; + m_call_uid = call_uid; + m_valid = true; + } - bool m_valid; - symbol_table *m_symbol_table; - function_info *m_func_info; - unsigned m_PC; - unsigned m_RPC; - const symbol *m_return_var_src; - const symbol *m_return_var_dst; - unsigned m_call_uid; + bool m_valid; + symbol_table *m_symbol_table; + function_info *m_func_info; + unsigned m_PC; + unsigned m_RPC; + const symbol *m_return_var_src; + const symbol *m_return_var_dst; + unsigned m_call_uid; }; class ptx_version { - public: - ptx_version() { - m_valid = false; - m_ptx_version = 0; - m_ptx_extensions = 0; - m_sm_version_valid = false; - m_texmode_unified = true; - m_map_f64_to_f32 = true; - } - ptx_version(float ver, unsigned extensions) { - m_valid = true; - m_ptx_version = ver; - m_ptx_extensions = extensions; - m_sm_version_valid = false; - m_texmode_unified = true; - } - void set_target(const char *sm_ver, const char *ext, const char *ext2) { - assert(m_valid); - m_sm_version_str = sm_ver; - check_target_extension(ext); - check_target_extension(ext2); - sscanf(sm_ver, "%u", &m_sm_version); - m_sm_version_valid = true; - } - float ver() const { - assert(m_valid); - return m_ptx_version; - } - unsigned target() const { - assert(m_valid && m_sm_version_valid); - return m_sm_version; - } - unsigned extensions() const { - assert(m_valid); - return m_ptx_extensions; - } - - private: - void check_target_extension(const char *ext) { - if (ext) { - if (!strcmp(ext, "texmode_independent")) - m_texmode_unified = false; - else if (!strcmp(ext, "texmode_unified")) - m_texmode_unified = true; - else if (!strcmp(ext, "map_f64_to_f32")) - m_map_f64_to_f32 = true; - else - abort(); - } - } +public: + ptx_version() + { + m_valid = false; + m_ptx_version = 0; + m_ptx_extensions = 0; + m_sm_version_valid=false; + m_texmode_unified=true; + m_map_f64_to_f32 = true; + } + ptx_version(float ver, unsigned extensions) + { + m_valid = true; + m_ptx_version = ver; + m_ptx_extensions = extensions; + m_sm_version_valid=false; + m_texmode_unified=true; + } + void set_target( const char *sm_ver, const char *ext, const char *ext2 ) + { + assert( m_valid ); + m_sm_version_str = sm_ver; + check_target_extension(ext); + check_target_extension(ext2); + sscanf(sm_ver,"%u",&m_sm_version); + m_sm_version_valid=true; + } + float ver() const { assert(m_valid); return m_ptx_version; } + unsigned target() const { assert(m_valid&&m_sm_version_valid); return m_sm_version; } + unsigned extensions() const { assert(m_valid); return m_ptx_extensions; } +private: + void check_target_extension( const char *ext ) + { + if( ext ) { + if( !strcmp(ext,"texmode_independent") ) + m_texmode_unified=false; + else if( !strcmp(ext,"texmode_unified") ) + m_texmode_unified=true; + else if( !strcmp(ext,"map_f64_to_f32") ) + m_map_f64_to_f32 = true; + else abort(); + } + } - bool m_valid; - float m_ptx_version; - unsigned m_sm_version_valid; - std::string m_sm_version_str; - bool m_texmode_unified; - bool m_map_f64_to_f32; - unsigned m_sm_version; - unsigned m_ptx_extensions; + bool m_valid; + float m_ptx_version; + unsigned m_sm_version_valid; + std::string m_sm_version_str; + bool m_texmode_unified; + bool m_map_f64_to_f32; + unsigned m_sm_version; + unsigned m_ptx_extensions; }; class ptx_thread_info { - public: - ~ptx_thread_info(); - ptx_thread_info(kernel_info_t &kernel); +public: + ~ptx_thread_info(); + ptx_thread_info( kernel_info_t &kernel ); - void init(gpgpu_t *gpu, core_t *core, unsigned sid, unsigned cta_id, - unsigned wid, unsigned tid, bool fsim) { - m_gpu = gpu; - m_core = core; - m_hw_sid = sid; - m_hw_ctaid = cta_id; - m_hw_wid = wid; - m_hw_tid = tid; - m_functionalSimulationMode = fsim; - } + void init(gpgpu_t *gpu, core_t *core, unsigned sid, unsigned cta_id, unsigned wid, unsigned tid, bool fsim) + { + m_gpu = gpu; + m_core = core; + m_hw_sid=sid; + m_hw_ctaid=cta_id; + m_hw_wid=wid; + m_hw_tid=tid; + m_functionalSimulationMode = fsim; + } - void ptx_fetch_inst(inst_t &inst) const; - void ptx_exec_inst(warp_inst_t &inst, unsigned lane_id); + void ptx_fetch_inst( inst_t &inst ) const; + void ptx_exec_inst( warp_inst_t &inst, unsigned lane_id ); - const ptx_version &get_ptx_version() const; - void set_reg(const symbol *reg, const ptx_reg_t &value); - void print_reg_thread(char *fname); - void resume_reg_thread(char *fname, symbol_table *symtab); - ptx_reg_t get_reg(const symbol *reg); - ptx_reg_t get_operand_value(const operand_info &op, operand_info dstInfo, - unsigned opType, ptx_thread_info *thread, - int derefFlag); - void set_operand_value(const operand_info &dst, const ptx_reg_t &data, - unsigned type, ptx_thread_info *thread, - const ptx_instruction *pI); - void set_operand_value(const operand_info &dst, const ptx_reg_t &data, - unsigned type, ptx_thread_info *thread, - const ptx_instruction *pI, int overflow, int carry); - void get_vector_operand_values(const operand_info &op, ptx_reg_t *ptx_regs, - unsigned num_elements); - void set_vector_operand_values(const operand_info &dst, - const ptx_reg_t &data1, const ptx_reg_t &data2, - const ptx_reg_t &data3, - const ptx_reg_t &data4); - void set_wmma_vector_operand_values( - const operand_info &dst, const ptx_reg_t &data1, const ptx_reg_t &data2, - const ptx_reg_t &data3, const ptx_reg_t &data4, const ptx_reg_t &data5, - const ptx_reg_t &data6, const ptx_reg_t &data7, const ptx_reg_t &data8); + const ptx_version &get_ptx_version() const; + void set_reg( const symbol *reg, const ptx_reg_t &value ); + void print_reg_thread (char * fname); + void resume_reg_thread(char * fname, symbol_table * symtab); + ptx_reg_t get_reg( const symbol *reg ); + ptx_reg_t get_operand_value( const operand_info &op, operand_info dstInfo, unsigned opType, ptx_thread_info *thread, int derefFlag ); + void set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI ); + void set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI, int overflow, int carry ); + void get_vector_operand_values( const operand_info &op, ptx_reg_t* ptx_regs, unsigned num_elements ); + void set_vector_operand_values( const operand_info &dst, + const ptx_reg_t &data1, + const ptx_reg_t &data2, + const ptx_reg_t &data3, + const ptx_reg_t &data4 ); + void set_wmma_vector_operand_values( const operand_info &dst, + const ptx_reg_t &data1, + const ptx_reg_t &data2, + const ptx_reg_t &data3, + const ptx_reg_t &data4, + const ptx_reg_t &data5, + const ptx_reg_t &data6, + const ptx_reg_t &data7, + const ptx_reg_t &data8 ); - function_info *func_info() { return m_func_info; } - void print_insn(unsigned pc, FILE *fp) const; - void set_info(function_info *func); - unsigned get_uid() const { return m_uid; } + function_info *func_info() + { + return m_func_info; + } + void print_insn( unsigned pc, FILE * fp ) const; + void set_info( function_info *func ); + unsigned get_uid() const + { + return m_uid; + } - dim3 get_ctaid() const { return m_ctaid; } - dim3 get_tid() const { return m_tid; } - dim3 get_ntid() const { return m_ntid; } - class gpgpu_sim *get_gpu() { - return (gpgpu_sim *)m_gpu; - } - unsigned get_hw_tid() const { return m_hw_tid; } - unsigned get_hw_ctaid() const { return m_hw_ctaid; } - unsigned get_hw_wid() const { return m_hw_wid; } - unsigned get_hw_sid() const { return m_hw_sid; } - core_t *get_core() { return m_core; } + dim3 get_ctaid() const { return m_ctaid; } + dim3 get_tid() const { return m_tid; } + dim3 get_ntid() const { return m_ntid; } + class gpgpu_sim *get_gpu() { return (gpgpu_sim*)m_gpu;} + unsigned get_hw_tid() const { return m_hw_tid;} + unsigned get_hw_ctaid() const { return m_hw_ctaid;} + unsigned get_hw_wid() const { return m_hw_wid;} + unsigned get_hw_sid() const { return m_hw_sid;} + core_t *get_core() { return m_core; } - unsigned get_icount() const { return m_icount; } - void set_valid() { m_valid = true; } - addr_t last_eaddr() const { return m_last_effective_address; } - memory_space_t last_space() const { return m_last_memory_space; } - dram_callback_t last_callback() const { return m_last_dram_callback; } - unsigned long long get_cta_uid() { return m_cta_info->get_sm_idx(); } + unsigned get_icount() const { return m_icount;} + void set_valid() { m_valid = true;} + addr_t last_eaddr() const { return m_last_effective_address;} + memory_space_t last_space() const { return m_last_memory_space;} + dram_callback_t last_callback() const { return m_last_dram_callback;} + unsigned long long get_cta_uid() { return m_cta_info->get_sm_idx();} - void set_single_thread_single_block() { - m_ntid.x = 1; - m_ntid.y = 1; - m_ntid.z = 1; - m_ctaid.x = 0; - m_ctaid.y = 0; - m_ctaid.z = 0; - m_tid.x = 0; - m_tid.y = 0; - m_tid.z = 0; - m_nctaid.x = 1; - m_nctaid.y = 1; - m_nctaid.z = 1; - m_gridid = 0; - m_valid = true; - } - void set_tid(dim3 tid) { m_tid = tid; } - void cpy_tid_to_reg(dim3 tid); - void set_ctaid(dim3 ctaid) { m_ctaid = ctaid; } - void set_ntid(dim3 tid) { m_ntid = tid; } - void set_nctaid(dim3 cta_size) { m_nctaid = cta_size; } + void set_single_thread_single_block() + { + m_ntid.x = 1; + m_ntid.y = 1; + m_ntid.z = 1; + m_ctaid.x = 0; + m_ctaid.y = 0; + m_ctaid.z = 0; + m_tid.x = 0; + m_tid.y = 0; + m_tid.z = 0; + m_nctaid.x = 1; + m_nctaid.y = 1; + m_nctaid.z = 1; + m_gridid = 0; + m_valid = true; + } + void set_tid( dim3 tid ) { m_tid = tid; } + void cpy_tid_to_reg( dim3 tid ); + void set_ctaid( dim3 ctaid ) { m_ctaid = ctaid; } + void set_ntid( dim3 tid ) { m_ntid = tid; } + void set_nctaid( dim3 cta_size ) { m_nctaid = cta_size; } - unsigned get_builtin(int builtin_id, unsigned dim_mod); + unsigned get_builtin( int builtin_id, unsigned dim_mod ); - void set_done(); - bool is_done() { return m_thread_done; } - unsigned donecycle() const { return m_cycle_done; } + void set_done(); + bool is_done() { return m_thread_done;} + unsigned donecycle() const { return m_cycle_done; } - unsigned next_instr() { - m_icount++; - m_branch_taken = false; - return m_PC; - } - bool branch_taken() const { return m_branch_taken; } - unsigned get_pc() const { return m_PC; } - void set_npc(unsigned npc) { m_NPC = npc; } - void set_npc(const function_info *f); - void callstack_push(unsigned npc, unsigned rpc, const symbol *return_var_src, - const symbol *return_var_dst, unsigned call_uid); - bool callstack_pop(); - void callstack_push_plus(unsigned npc, unsigned rpc, - const symbol *return_var_src, - const symbol *return_var_dst, unsigned call_uid); - bool callstack_pop_plus(); - void dump_callstack() const; - std::string get_location() const; - const ptx_instruction *get_inst() const; - const ptx_instruction *get_inst(addr_t pc) const; - bool rpc_updated() const { return m_RPC_updated; } - bool last_was_call() const { return m_last_was_call; } - unsigned get_rpc() const { return m_RPC; } - void clearRPC() { - m_RPC = -1; - m_RPC_updated = false; - m_last_was_call = false; - } - unsigned get_return_PC() { return m_callstack.back().m_PC; } - void update_pc() { m_PC = m_NPC; } - void dump_regs(FILE *fp); - void dump_modifiedregs(FILE *fp); - void clear_modifiedregs() { - m_debug_trace_regs_modified.back().clear(); - m_debug_trace_regs_read.back().clear(); - } - function_info *get_finfo() { return m_func_info; } - const function_info *get_finfo() const { return m_func_info; } - void push_breakaddr(const operand_info &breakaddr); - const operand_info &pop_breakaddr(); - void enable_debug_trace() { m_enable_debug_trace = true; } - unsigned get_local_mem_stack_pointer() const { - return m_local_mem_stack_pointer; - } + unsigned next_instr() + { + m_icount++; + m_branch_taken = false; + return m_PC; + } + bool branch_taken() const + { + return m_branch_taken; + } + unsigned get_pc() const + { + return m_PC; + } + void set_npc( unsigned npc ) + { + m_NPC = npc; + } + void set_npc( const function_info *f ); + void callstack_push( unsigned npc, unsigned rpc, const symbol *return_var_src, const symbol *return_var_dst, unsigned call_uid ); + bool callstack_pop(); + void callstack_push_plus( unsigned npc, unsigned rpc, const symbol *return_var_src, const symbol *return_var_dst, unsigned call_uid ); + bool callstack_pop_plus(); + void dump_callstack() const; + std::string get_location() const; + const ptx_instruction *get_inst() const; + const ptx_instruction *get_inst( addr_t pc ) const; + bool rpc_updated() const { return m_RPC_updated; } + bool last_was_call() const { return m_last_was_call; } + unsigned get_rpc() const { return m_RPC; } + void clearRPC() + { + m_RPC = -1; + m_RPC_updated = false; + m_last_was_call = false; + } + unsigned get_return_PC() + { + return m_callstack.back().m_PC; + } + void update_pc( ) + { + m_PC = m_NPC; + } + void dump_regs(FILE * fp); + void dump_modifiedregs(FILE *fp); + void clear_modifiedregs() { m_debug_trace_regs_modified.back().clear(); m_debug_trace_regs_read.back().clear(); } + function_info *get_finfo() { return m_func_info; } + const function_info *get_finfo() const { return m_func_info; } + void push_breakaddr(const operand_info &breakaddr); + const operand_info& pop_breakaddr(); + void enable_debug_trace() { m_enable_debug_trace = true; } + unsigned get_local_mem_stack_pointer() const { return m_local_mem_stack_pointer; } - memory_space *get_global_memory() { return m_gpu->get_global_memory(); } - memory_space *get_tex_memory() { return m_gpu->get_tex_memory(); } - memory_space *get_surf_memory() { return m_gpu->get_surf_memory(); } - memory_space *get_param_memory() { return m_kernel.get_param_memory(); } - const gpgpu_functional_sim_config &get_config() const { - return m_gpu->get_config(); - } - bool isInFunctionalSimulationMode() { return m_functionalSimulationMode; } - void exitCore() { - // m_core is not used in case of functional simulation mode - if (!m_functionalSimulationMode) m_core->warp_exit(m_hw_wid); - } + memory_space *get_global_memory() { return m_gpu->get_global_memory(); } + memory_space *get_tex_memory() { return m_gpu->get_tex_memory(); } + memory_space *get_surf_memory() { return m_gpu->get_surf_memory(); } + memory_space *get_param_memory() { return m_kernel.get_param_memory(); } + const gpgpu_functional_sim_config &get_config() const { return m_gpu->get_config(); } + bool isInFunctionalSimulationMode(){ return m_functionalSimulationMode;} + void exitCore() + { + //m_core is not used in case of functional simulation mode + if(!m_functionalSimulationMode) + m_core->warp_exit(m_hw_wid); + } + + void registerExit(){m_cta_info->register_thread_exit(this);} + unsigned get_reduction_value(unsigned ctaid, unsigned barid) {return m_core->get_reduction_value(ctaid,barid);} + void and_reduction(unsigned ctaid, unsigned barid, bool value) {m_core->and_reduction(ctaid,barid,value);} + void or_reduction(unsigned ctaid, unsigned barid, bool value) {m_core->or_reduction(ctaid,barid,value);} + void popc_reduction(unsigned ctaid, unsigned barid, bool value) {m_core->popc_reduction(ctaid,barid,value);} - void registerExit() { m_cta_info->register_thread_exit(this); } - unsigned get_reduction_value(unsigned ctaid, unsigned barid) { - return m_core->get_reduction_value(ctaid, barid); - } - void and_reduction(unsigned ctaid, unsigned barid, bool value) { - m_core->and_reduction(ctaid, barid, value); - } - void or_reduction(unsigned ctaid, unsigned barid, bool value) { - m_core->or_reduction(ctaid, barid, value); - } - void popc_reduction(unsigned ctaid, unsigned barid, bool value) { - m_core->popc_reduction(ctaid, barid, value); - } + //Jin: get corresponding kernel grid for CDP purpose + kernel_info_t & get_kernel() { return m_kernel; } - // Jin: get corresponding kernel grid for CDP purpose - kernel_info_t &get_kernel() { return m_kernel; } +public: + addr_t m_last_effective_address; + bool m_branch_taken; + memory_space_t m_last_memory_space; + dram_callback_t m_last_dram_callback; + memory_space *m_shared_mem; + memory_space *m_sstarr_mem; + memory_space *m_local_mem; + ptx_warp_info *m_warp_info; + ptx_cta_info *m_cta_info; + ptx_reg_t m_last_set_operand_value; - public: - addr_t m_last_effective_address; - bool m_branch_taken; - memory_space_t m_last_memory_space; - dram_callback_t m_last_dram_callback; - memory_space *m_shared_mem; - memory_space *m_sstarr_mem; - memory_space *m_local_mem; - ptx_warp_info *m_warp_info; - ptx_cta_info *m_cta_info; - ptx_reg_t m_last_set_operand_value; +private: - private: - bool m_functionalSimulationMode; - unsigned m_uid; - kernel_info_t &m_kernel; - core_t *m_core; - gpgpu_t *m_gpu; - bool m_valid; - dim3 m_ntid; - dim3 m_tid; - dim3 m_nctaid; - dim3 m_ctaid; - unsigned m_gridid; - bool m_thread_done; - unsigned m_hw_sid; - unsigned m_hw_tid; - unsigned m_hw_wid; - unsigned m_hw_ctaid; + bool m_functionalSimulationMode; + unsigned m_uid; + kernel_info_t &m_kernel; + core_t *m_core; + gpgpu_t *m_gpu; + bool m_valid; + dim3 m_ntid; + dim3 m_tid; + dim3 m_nctaid; + dim3 m_ctaid; + unsigned m_gridid; + bool m_thread_done; + unsigned m_hw_sid; + unsigned m_hw_tid; + unsigned m_hw_wid; + unsigned m_hw_ctaid; - unsigned m_icount; - unsigned m_PC; - unsigned m_NPC; - unsigned m_RPC; - bool m_RPC_updated; - bool m_last_was_call; - unsigned m_cycle_done; + unsigned m_icount; + unsigned m_PC; + unsigned m_NPC; + unsigned m_RPC; + bool m_RPC_updated; + bool m_last_was_call; + unsigned m_cycle_done; - int m_barrier_num; - bool m_at_barrier; + int m_barrier_num; + bool m_at_barrier; - symbol_table *m_symbol_table; - function_info *m_func_info; + symbol_table *m_symbol_table; + function_info *m_func_info; - std::list<stack_entry> m_callstack; - unsigned m_local_mem_stack_pointer; + std::list<stack_entry> m_callstack; + unsigned m_local_mem_stack_pointer; - typedef tr1_hash_map<const symbol *, ptx_reg_t> reg_map_t; - std::list<reg_map_t> m_regs; - std::list<reg_map_t> m_debug_trace_regs_modified; - std::list<reg_map_t> m_debug_trace_regs_read; - bool m_enable_debug_trace; + typedef tr1_hash_map<const symbol*,ptx_reg_t> reg_map_t; + std::list<reg_map_t> m_regs; + std::list<reg_map_t> m_debug_trace_regs_modified; + std::list<reg_map_t> m_debug_trace_regs_read; + bool m_enable_debug_trace; - std::stack<class operand_info, std::vector<operand_info> > m_breakaddrs; + std::stack<class operand_info, std::vector<operand_info> > m_breakaddrs; }; -addr_t generic_to_local(unsigned smid, unsigned hwtid, addr_t addr); -addr_t generic_to_shared(unsigned smid, addr_t addr); -addr_t generic_to_global(addr_t addr); -addr_t local_to_generic(unsigned smid, unsigned hwtid, addr_t addr); -addr_t shared_to_generic(unsigned smid, addr_t addr); -addr_t global_to_generic(addr_t addr); -bool isspace_local(unsigned smid, unsigned hwtid, addr_t addr); -bool isspace_shared(unsigned smid, addr_t addr); -bool isspace_global(addr_t addr); -memory_space_t whichspace(addr_t addr); +addr_t generic_to_local( unsigned smid, unsigned hwtid, addr_t addr ); +addr_t generic_to_shared( unsigned smid, addr_t addr ); +addr_t generic_to_global( addr_t addr ); +addr_t local_to_generic( unsigned smid, unsigned hwtid, addr_t addr ); +addr_t shared_to_generic( unsigned smid, addr_t addr ); +addr_t global_to_generic( addr_t addr ); +bool isspace_local( unsigned smid, unsigned hwtid, addr_t addr ); +bool isspace_shared( unsigned smid, addr_t addr ); +bool isspace_global( addr_t addr ); +memory_space_t whichspace( addr_t addr ); extern unsigned g_ptx_thread_info_uid_next; |
