From 48c9d92e2be9a9fe264d3783b0b3ee7af8295b53 Mon Sep 17 00:00:00 2001 From: Tor Aamodt Date: Sun, 1 Apr 2018 16:33:59 -0700 Subject: add pdom analysis for function calls -- doesn't fix regressions --- src/cuda-sim/instructions.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'src/cuda-sim/instructions.cc') diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 71286c9..5d97287 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -1466,7 +1466,14 @@ void call_impl( const ptx_instruction *pI, ptx_thread_info *thread ) const operand_info &target = pI->func_addr(); assert( target.is_function_address() ); const symbol *func_addr = target.get_symbol(); - const function_info *target_func = func_addr->get_pc(); + function_info *target_func = func_addr->get_pc(); + if (target_func->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", target_func->get_name().c_str() ); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", target_func->get_name().c_str() ); + target_func->do_pdom(); + target_func->set_pdom(); + } // check that number of args and return match function requirements if( pI->has_return() ^ target_func->has_return() ) { -- cgit v1.3 From 26476592e3650e796b51c94dd1a25c162eb1aa64 Mon Sep 17 00:00:00 2001 From: Amruth Date: Tue, 3 Apr 2018 11:43:46 -0700 Subject: crash when print() is sent to pdom analysis --- libcuda/cuda_runtime_api.cc | 15 +- libcuda/cuda_runtime_api.cc~ | 2515 +++++++++++++++++++++++ src/cuda-sim/cuda-sim.cc~ | 2155 ++++++++++++++++++++ src/cuda-sim/instructions.cc | 7 +- src/cuda-sim/instructions.cc~ | 4517 +++++++++++++++++++++++++++++++++++++++++ src/cuda-sim/ptx_loader.cc~ | 462 +++++ 6 files changed, 9663 insertions(+), 8 deletions(-) create mode 100644 libcuda/cuda_runtime_api.cc~ create mode 100644 src/cuda-sim/cuda-sim.cc~ create mode 100644 src/cuda-sim/instructions.cc~ create mode 100644 src/cuda-sim/ptx_loader.cc~ (limited to 'src/cuda-sim/instructions.cc') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index 6cf21dd..ded1aee 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -1499,6 +1499,12 @@ void extract_code_using_cuobjdump(){ no_of_ptx = no_of_ptx + 1; fclose(fp); } + if(no_of_ptx==0){ + printf("WARNING: Number of ptx in the executable file are 0. One of the reasons might be\n"); + printf("\t1. CDP is enabled\n"); + printf("\t2. cuobjdump -lptx doesnt recognize sm_%u\n",forced_max_capability); + printf("\t3. the application was not compiled with nvcc flag sm_%u\n",forced_max_capability); + } } if(!g_cdp_enabled) { //based on the list above, dump ptx files individually. Format of dumped ptx file is prog_name.unique_no.sm_<>.ptx @@ -1520,15 +1526,10 @@ void extract_code_using_cuobjdump(){ snprintf(fname,1024,"_cuobjdump_complete_output_XXXXXX"); int fd=mkstemp(fname); close(fd); - if(!g_cdp_enabled) { -#if (CUDART_VERSION >= 6000) - snprintf(command,1000,"$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass -arch=sm_%u %s > %s", forced_max_capability, app_binary.c_str(), fname); -#else + if(!g_cdp_enabled) snprintf(command,1000,"$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass %s > %s", app_binary.c_str(), fname); -#endif - } else { + else snprintf(command,1000,"$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass -all %s > %s", app_binary.c_str(), fname); - } bool parse_output = true; result = system(command); if(result) { diff --git a/libcuda/cuda_runtime_api.cc~ b/libcuda/cuda_runtime_api.cc~ new file mode 100644 index 0000000..de7f5e9 --- /dev/null +++ b/libcuda/cuda_runtime_api.cc~ @@ -0,0 +1,2515 @@ +// This file created from cuda_runtime_api.h distributed with CUDA 1.1 +// Changes Copyright 2009, Tor M. Aamodt, Ali Bakhoda and George L. Yuan +// University of British Columbia + +/* + * cuda_runtime_api.cc + * + * Copyright © 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * George L. Yuan and the University of British Columbia, Vancouver, + * BC V6T 1Z4, All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +/* + * Copyright 1993-2007 NVIDIA Corporation. All rights reserved. + * + * NOTICE TO USER: + * + * This source code is subject to NVIDIA ownership rights under U.S. and + * international Copyright laws. Users and possessors of this source code + * are hereby granted a nonexclusive, royalty-free license to use this code + * in individual and commercial software. + * + * NVIDIA MAKES NO REPRESENTATION ABOUT THE SUITABILITY OF THIS SOURCE + * CODE FOR ANY PURPOSE. IT IS PROVIDED "AS IS" WITHOUT EXPRESS OR + * IMPLIED WARRANTY OF ANY KIND. NVIDIA DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOURCE CODE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY, NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. + * IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL, + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE + * OR PERFORMANCE OF THIS SOURCE CODE. + * + * U.S. Government End Users. This source code is a "commercial item" as + * that term is defined at 48 C.F.R. 2.101 (OCT 1995), consisting of + * "commercial computer software" and "commercial computer software + * documentation" as such terms are used in 48 C.F.R. 12.212 (SEPT 1995) + * and is provided to the U.S. Government only as a commercial end item. + * Consistent with 48 C.F.R.12.212 and 48 C.F.R. 227.7202-1 through + * 227.7202-4 (JUNE 1995), all U.S. Government End Users acquire the + * source code with only those rights set forth herein. + * + * Any use of this source code in individual and commercial software must + * include, in the user documentation and internal comments to the code, + * the above Disclaimer and U.S. Government End Users Notice. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef OPENGL_SUPPORT +#define GL_GLEXT_PROTOTYPES +#ifdef __APPLE__ +#include // Apple's version of GLUT is here +#else +#include +#endif +#endif + +#define __CUDA_RUNTIME_API_H__ + +#include "host_defines.h" +#include "builtin_types.h" +#include "driver_types.h" +#if (CUDART_VERSION < 8000) +#include "__cudaFatFormat.h" +#endif +#include "../src/gpgpu-sim/gpu-sim.h" +#include "../src/cuda-sim/ptx_loader.h" +#include "../src/cuda-sim/cuda-sim.h" +#include "../src/cuda-sim/ptx_ir.h" +#include "../src/cuda-sim/ptx_parser.h" +#include "../src/gpgpusim_entrypoint.h" +#include "../src/stream_manager.h" +#include "../src/abstract_hardware_model.h" + +#include +#include + +#ifdef __APPLE__ +#include +#endif + +std::map pinned_memory; //support for pinned memories added +std::map pinned_memory_size; +int no_of_ptx=0; + +extern void synchronize(); +extern void exit_simulation(); + +static int load_static_globals( symbol_table *symtab, unsigned min_gaddr, unsigned max_gaddr, gpgpu_t *gpu ); +static int load_constants( symbol_table *symtab, addr_t min_gaddr, gpgpu_t *gpu ); + +static kernel_info_t *gpgpu_cuda_ptx_sim_init_grid( const char *kernel_key, + gpgpu_ptx_sim_arg_list_t args, + struct dim3 gridDim, + struct dim3 blockDim, + struct CUctx_st* context ); + +/*DEVICE_BUILTIN*/ +struct cudaArray +{ + void *devPtr; + int devPtr32; + struct cudaChannelFormatDesc desc; + int width; + int height; + int size; //in bytes + unsigned dimensions; +}; + +#if !defined(__dv) +#if defined(__cplusplus) +#define __dv(v) \ + = v +#else /* __cplusplus */ +#define __dv(v) +#endif /* __cplusplus */ +#endif /* !__dv */ + +cudaError_t g_last_cudaError = cudaSuccess; + +extern stream_manager *g_stream_manager; + +void register_ptx_function( const char *name, function_info *impl ) +{ + // no longer need this +} + +#if defined __APPLE__ +# define __my_func__ __PRETTY_FUNCTION__ +#else +# if defined __cplusplus ? __GNUC_PREREQ (2, 6) : __GNUC_PREREQ (2, 4) +# define __my_func__ __PRETTY_FUNCTION__ +# else +# if defined __STDC_VERSION__ && __STDC_VERSION__ >= 199901L +# define __my_func__ __func__ +# else +# define __my_func__ ((__const char *) 0) +# endif +# endif +#endif + +struct _cuda_device_id { + _cuda_device_id(gpgpu_sim* gpu) {m_id = 0; m_next = NULL; m_gpgpu=gpu;} + struct _cuda_device_id *next() { return m_next; } + unsigned num_shader() const { return m_gpgpu->get_config().num_shader(); } + int num_devices() const { + if( m_next == NULL ) return 1; + else return 1 + m_next->num_devices(); + } + struct _cuda_device_id *get_device( unsigned n ) + { + assert( n < (unsigned)num_devices() ); + struct _cuda_device_id *p=this; + for(unsigned i=0; im_next; + return p; + } + const struct cudaDeviceProp *get_prop() const + { + return m_gpgpu->get_prop(); + } + unsigned get_id() const { return m_id; } + + gpgpu_sim *get_gpgpu() { return m_gpgpu; } +private: + unsigned m_id; + class gpgpu_sim *m_gpgpu; + struct _cuda_device_id *m_next; +}; + +struct CUctx_st { + CUctx_st( _cuda_device_id *gpu ) + { + m_gpu = gpu; + m_binary_info.cmem = 0; + m_binary_info.gmem = 0; + } + + _cuda_device_id *get_device() { return m_gpu; } + + void add_binary( symbol_table *symtab, unsigned fat_cubin_handle ) + { + m_code[fat_cubin_handle] = symtab; + m_last_fat_cubin_handle = fat_cubin_handle; + } + + void add_ptxinfo( const char *deviceFun, const struct gpgpu_ptx_sim_info &info ) + { + symbol *s = m_code[m_last_fat_cubin_handle]->lookup(deviceFun); + assert( s != NULL ); + function_info *f = s->get_pc(); + assert( f != NULL ); + f->set_kernel_info(info); + } + + void add_ptxinfo( const struct gpgpu_ptx_sim_info &info ) + { + m_binary_info = info; + } + + void register_function( unsigned fat_cubin_handle, const char *hostFun, const char *deviceFun ) + { + if( m_code.find(fat_cubin_handle) != m_code.end() ) { + symbol *s = m_code[fat_cubin_handle]->lookup(deviceFun); + if(s != NULL) { + function_info *f = s->get_pc(); + assert( f != NULL ); + m_kernel_lookup[hostFun] = f; + } + else { + printf("Warning: cannot find deviceFun %s\n", deviceFun); + m_kernel_lookup[hostFun] = NULL; + } + // assert( s != NULL ); + // function_info *f = s->get_pc(); + // assert( f != NULL ); + // m_kernel_lookup[hostFun] = f; + } else { + m_kernel_lookup[hostFun] = NULL; + } + } + + function_info *get_kernel(const char *hostFun) + { + std::map::iterator i=m_kernel_lookup.find(hostFun); + assert( i != m_kernel_lookup.end() ); + return i->second; + } + +private: + _cuda_device_id *m_gpu; // selected gpu + std::map m_code; // fat binary handle => global symbol table + unsigned m_last_fat_cubin_handle; + std::map m_kernel_lookup; // unique id (CUDA app function address) => kernel entry point + struct gpgpu_ptx_sim_info m_binary_info; + +}; + +class kernel_config { +public: + kernel_config( dim3 GridDim, dim3 BlockDim, size_t sharedMem, struct CUstream_st *stream ) + { + m_GridDim=GridDim; + m_BlockDim=BlockDim; + m_sharedMem=sharedMem; + m_stream = stream; + } + void set_arg( const void *arg, size_t size, size_t offset ) + { + m_args.push_front( gpgpu_ptx_sim_arg(arg,size,offset) ); + } + dim3 grid_dim() const { return m_GridDim; } + dim3 block_dim() const { return m_BlockDim; } + gpgpu_ptx_sim_arg_list_t get_args() { return m_args; } + struct CUstream_st *get_stream() { return m_stream; } + +private: + dim3 m_GridDim; + dim3 m_BlockDim; + size_t m_sharedMem; + struct CUstream_st *m_stream; + gpgpu_ptx_sim_arg_list_t m_args; +}; + +class _cuda_device_id *GPGPUSim_Init() +{ + static _cuda_device_id *the_device = NULL; + if( !the_device ) { + gpgpu_sim *the_gpu = gpgpu_ptx_sim_init_perf(); + + cudaDeviceProp *prop = (cudaDeviceProp *) calloc(sizeof(cudaDeviceProp),1); + snprintf(prop->name,256,"GPGPU-Sim_v%s", g_gpgpusim_version_string ); + prop->major = 5; + prop->minor = 2; + prop->totalGlobalMem = 0x80000000 /* 2 GB */; + prop->memPitch = 0; + prop->maxThreadsPerBlock = 512; + prop->maxThreadsDim[0] = 512; + prop->maxThreadsDim[1] = 512; + prop->maxThreadsDim[2] = 512; + prop->maxGridSize[0] = 0x40000000; + prop->maxGridSize[1] = 0x40000000; + prop->maxGridSize[2] = 0x40000000; + prop->totalConstMem = 0x40000000; + prop->textureAlignment = 0; + prop->sharedMemPerBlock = the_gpu->shared_mem_size(); + prop->regsPerBlock = the_gpu->num_registers_per_core(); + prop->warpSize = the_gpu->wrp_size(); + prop->clockRate = the_gpu->shader_clock(); +#if (CUDART_VERSION >= 2010) + prop->multiProcessorCount = the_gpu->get_config().num_shader(); +#endif + the_gpu->set_prop(prop); + the_device = new _cuda_device_id(the_gpu); + } + start_sim_thread(1); + return the_device; +} + +static CUctx_st* GPGPUSim_Context() +{ + static CUctx_st *the_context = NULL; + if( the_context == NULL ) { + _cuda_device_id *the_gpu = GPGPUSim_Init(); + the_context = new CUctx_st(the_gpu); + } + return the_context; +} + + void ptxinfo_addinfo() +{ + if(!get_ptxinfo_kname()){ + /* This info is not per kernel (since CUDA 5.0 some info (e.g. gmem, and cmem) is added at the beginning for the whole binary ) */ + CUctx_st *context = GPGPUSim_Context(); + print_ptxinfo(); + context->add_ptxinfo(get_ptxinfo()); + clear_ptxinfo(); + return; + } + if( !strcmp("__cuda_dummy_entry__",get_ptxinfo_kname()) ) { + // this string produced by ptxas for empty ptx files (e.g., bandwidth test) + clear_ptxinfo(); + return; + } + CUctx_st *context = GPGPUSim_Context(); + print_ptxinfo(); + context->add_ptxinfo( get_ptxinfo_kname(), get_ptxinfo() ); + clear_ptxinfo(); +} + +void cuda_not_implemented( const char* func, unsigned line ) +{ + fflush(stdout); + fflush(stderr); + printf("\n\nGPGPU-Sim PTX: Execution error: CUDA API function \"%s()\" has not been implemented yet.\n" + " [$GPGPUSIM_ROOT/libcuda/%s around line %u]\n\n\n", + func,__FILE__, line ); + fflush(stdout); + abort(); +} + + +#define gpgpusim_ptx_error(msg, ...) gpgpusim_ptx_error_impl(__func__, __FILE__,__LINE__, msg, ##__VA_ARGS__) +#define gpgpusim_ptx_assert(cond,msg, ...) gpgpusim_ptx_assert_impl((cond),__func__, __FILE__,__LINE__, msg, ##__VA_ARGS__) + +void gpgpusim_ptx_error_impl( const char *func, const char *file, unsigned line, const char *msg, ... ) +{ + va_list ap; + char buf[1024]; + va_start(ap,msg); + vsnprintf(buf,1024,msg,ap); + va_end(ap); + + printf("GPGPU-Sim CUDA API: %s\n", buf); + printf(" [%s:%u : %s]\n", file, line, func ); + abort(); +} + +void gpgpusim_ptx_assert_impl( int test_value, const char *func, const char *file, unsigned line, const char *msg, ... ) +{ + va_list ap; + char buf[1024]; + va_start(ap,msg); + vsnprintf(buf,1024,msg,ap); + va_end(ap); + + if ( test_value == 0 ) + gpgpusim_ptx_error_impl(func, file, line, msg); +} + + +typedef std::map event_tracker_t; + +int CUevent_st::m_next_event_uid; +event_tracker_t g_timer_events; +int g_active_device = 0; //active gpu that runs the code +std::list g_cuda_launch_stack; + +/******************************************************************************* + * * + * * + * * + *******************************************************************************/ + +extern "C" { + +/******************************************************************************* + * * + * * + * * + *******************************************************************************/ +cudaError_t cudaPeekAtLastError(void) +{ + return g_last_cudaError; +} + +__host__ cudaError_t CUDARTAPI cudaMalloc(void **devPtr, size_t size) +{ + CUctx_st* context = GPGPUSim_Context(); + *devPtr = context->get_device()->get_gpgpu()->gpu_malloc(size); + if(g_debug_execution >= 3) + printf("GPGPU-Sim PTX: cudaMallocing %zu bytes starting at 0x%llx..\n",size, (unsigned long long) *devPtr); + if ( *devPtr ) { + return g_last_cudaError = cudaSuccess; + } else { + return g_last_cudaError = cudaErrorMemoryAllocation; + } +} + +__host__ cudaError_t CUDARTAPI cudaMallocHost(void **ptr, size_t size) +{ + GPGPUSim_Context(); + *ptr = malloc(size); + if ( *ptr ) { + //track pinned memory size allocated in the host so that same amount of memory is also allocated in GPU. + pinned_memory_size[*ptr]=size; + return g_last_cudaError = cudaSuccess; + } else { + return g_last_cudaError = cudaErrorMemoryAllocation; + } +} +__host__ cudaError_t CUDARTAPI cudaMallocPitch(void **devPtr, size_t *pitch, size_t width, size_t height) +{ + unsigned malloc_width_inbytes = width; + printf("GPGPU-Sim PTX: cudaMallocPitch (width = %d)\n", malloc_width_inbytes); + CUctx_st* ctx = GPGPUSim_Context(); + *devPtr = ctx->get_device()->get_gpgpu()->gpu_malloc(malloc_width_inbytes*height); + pitch[0] = malloc_width_inbytes; + if ( *devPtr ) { + return g_last_cudaError = cudaSuccess; + } else { + return g_last_cudaError = cudaErrorMemoryAllocation; + } +} + +__host__ cudaError_t CUDARTAPI cudaMallocArray(struct cudaArray **array, const struct cudaChannelFormatDesc *desc, size_t width, size_t height __dv(1)) +{ + unsigned size = width * height * ((desc->x + desc->y + desc->z + desc->w)/8); + CUctx_st* context = GPGPUSim_Context(); + (*array) = (struct cudaArray*) malloc(sizeof(struct cudaArray)); + (*array)->desc = *desc; + (*array)->width = width; + (*array)->height = height; + (*array)->size = size; + (*array)->dimensions = 2; + ((*array)->devPtr32)= (int) (long long)context->get_device()->get_gpgpu()->gpu_mallocarray(size); + printf("GPGPU-Sim PTX: cudaMallocArray: devPtr32 = %d\n", ((*array)->devPtr32)); + ((*array)->devPtr) = (void*) (long long) ((*array)->devPtr32); + if ( ((*array)->devPtr) ) { + return g_last_cudaError = cudaSuccess; + } else { + return g_last_cudaError = cudaErrorMemoryAllocation; + } +} + +__host__ cudaError_t CUDARTAPI cudaFree(void *devPtr) +{ + // TODO... manage g_global_mem space? + return g_last_cudaError = cudaSuccess; +} +__host__ cudaError_t CUDARTAPI cudaFreeHost(void *ptr) +{ + free (ptr); // this will crash the system if called twice + return g_last_cudaError = cudaSuccess; +} + +__host__ cudaError_t CUDARTAPI cudaFreeArray(struct cudaArray *array) +{ + // TODO... manage g_global_mem space? + return g_last_cudaError = cudaSuccess; +}; + + +/******************************************************************************* + * * + * * + * * + *******************************************************************************/ + +__host__ cudaError_t CUDARTAPI cudaMemcpy(void *dst, const void *src, size_t count, enum cudaMemcpyKind kind) +{ + //CUctx_st *context = GPGPUSim_Context(); + //gpgpu_t *gpu = context->get_device()->get_gpgpu(); + if(g_debug_execution >= 3) + printf("GPGPU-Sim PTX: cudaMemcpy(): devPtr = %p\n", dst); + if( kind == cudaMemcpyHostToDevice ) + g_stream_manager->push( stream_operation(src,(size_t)dst,count,0) ); + else if( kind == cudaMemcpyDeviceToHost ) + g_stream_manager->push( stream_operation((size_t)src,dst,count,0) ); + else if( kind == cudaMemcpyDeviceToDevice ) + g_stream_manager->push( stream_operation((size_t)src,(size_t)dst,count,0) ); + else if ( kind == cudaMemcpyDefault ) { + if ((size_t)src >= GLOBAL_HEAP_START) { + if ((size_t)dst >= GLOBAL_HEAP_START) + g_stream_manager->push( stream_operation((size_t)src,(size_t)dst,count,0) ); // device to device + else + g_stream_manager->push( stream_operation((size_t)src,dst,count,0) ); // device to host + } + else { + if ((size_t)dst >= GLOBAL_HEAP_START) + g_stream_manager->push( stream_operation(src,(size_t)dst,count,0) ); + else { + printf("GPGPU-Sim PTX: cudaMemcpy - ERROR : unsupported transfer: host to host\n"); + abort(); + } + } + } + else { + printf("GPGPU-Sim PTX: cudaMemcpy - ERROR : unsupported cudaMemcpyKind\n"); + abort(); + } + return g_last_cudaError = cudaSuccess; +} + +__host__ cudaError_t CUDARTAPI cudaMemcpyToArray(struct cudaArray *dst, size_t wOffset, size_t hOffset, const void *src, size_t count, enum cudaMemcpyKind kind) +{ + CUctx_st *context = GPGPUSim_Context(); + gpgpu_t *gpu = context->get_device()->get_gpgpu(); + size_t size = count; + printf("GPGPU-Sim PTX: cudaMemcpyToArray\n"); + if( kind == cudaMemcpyHostToDevice ) + gpu->memcpy_to_gpu( (size_t)(dst->devPtr), src, size); + else if( kind == cudaMemcpyDeviceToHost ) + gpu->memcpy_from_gpu( dst->devPtr, (size_t)src, size); + else if( kind == cudaMemcpyDeviceToDevice ) + gpu->memcpy_gpu_to_gpu( (size_t)(dst->devPtr), (size_t)src, size); + else { + printf("GPGPU-Sim PTX: cudaMemcpyToArray - ERROR : unsupported cudaMemcpyKind\n"); + abort(); + } + dst->devPtr32 = (unsigned) (size_t)(dst->devPtr); + return g_last_cudaError = cudaSuccess; +} + + +__host__ cudaError_t CUDARTAPI cudaMemcpyFromArray(void *dst, const struct cudaArray *src, size_t wOffset, size_t hOffset, size_t count, enum cudaMemcpyKind kind) +{ + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; +} + + +__host__ cudaError_t CUDARTAPI cudaMemcpyArrayToArray(struct cudaArray *dst, size_t wOffsetDst, size_t hOffsetDst, const struct cudaArray *src, size_t wOffsetSrc, size_t hOffsetSrc, size_t count, enum cudaMemcpyKind kind __dv(cudaMemcpyDeviceToDevice)) +{ + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; +} + + +__host__ cudaError_t CUDARTAPI cudaMemcpy2D(void *dst, size_t dpitch, const void *src, size_t spitch, size_t width, size_t height, enum cudaMemcpyKind kind) +{ + CUctx_st *context = GPGPUSim_Context(); + gpgpu_t *gpu = context->get_device()->get_gpgpu(); + size_t size = spitch*height; + gpgpusim_ptx_assert( (dpitch==spitch), "different src and dst pitch not supported yet" ); + if( kind == cudaMemcpyHostToDevice ) + gpu->memcpy_to_gpu( (size_t)dst, src, size ); + else if( kind == cudaMemcpyDeviceToHost ) + gpu->memcpy_from_gpu( dst, (size_t)src, size ); + else if( kind == cudaMemcpyDeviceToDevice ) + gpu->memcpy_gpu_to_gpu( (size_t)dst, (size_t)src, size); + else { + printf("GPGPU-Sim PTX: cudaMemcpy2D - ERROR : unsupported cudaMemcpyKind\n"); + abort(); + } + return g_last_cudaError = cudaSuccess; +} + + +__host__ cudaError_t CUDARTAPI cudaMemcpy2DToArray(struct cudaArray *dst, size_t wOffset, size_t hOffset, const void *src, size_t spitch, size_t width, size_t height, enum cudaMemcpyKind kind) +{ + CUctx_st *context = GPGPUSim_Context(); + gpgpu_t *gpu = context->get_device()->get_gpgpu(); + size_t size = spitch*height; + size_t channel_size = dst->desc.w+dst->desc.x+dst->desc.y+dst->desc.z; + gpgpusim_ptx_assert( ((channel_size%8) == 0), "none byte multiple destination channel size not supported (sz=%u)", channel_size ); + unsigned elem_size = channel_size/8; + gpgpusim_ptx_assert( (dst->dimensions==2), "copy to none 2D array not supported" ); + gpgpusim_ptx_assert( (wOffset==0), "non-zero wOffset not yet supported" ); + gpgpusim_ptx_assert( (hOffset==0), "non-zero hOffset not yet supported" ); + gpgpusim_ptx_assert( (dst->height == (int)height), "partial copy not supported" ); + gpgpusim_ptx_assert( (elem_size*dst->width == width), "partial copy not supported" ); + gpgpusim_ptx_assert( (spitch == width), "spitch != width not supported" ); + if( kind == cudaMemcpyHostToDevice ) + gpu->memcpy_to_gpu( (size_t)(dst->devPtr), src, size); + else if( kind == cudaMemcpyDeviceToHost ) + gpu->memcpy_from_gpu( dst->devPtr, (size_t)src, size); + else if( kind == cudaMemcpyDeviceToDevice ) + gpu->memcpy_gpu_to_gpu( (size_t)dst->devPtr, (size_t)src, size); + else { + printf("GPGPU-Sim PTX: cudaMemcpy2D - ERROR : unsupported cudaMemcpyKind\n"); + abort(); + } + dst->devPtr32 = (unsigned) (size_t)(dst->devPtr); + return g_last_cudaError = cudaSuccess; +} + + +__host__ cudaError_t CUDARTAPI cudaMemcpy2DFromArray(void *dst, size_t dpitch, const struct cudaArray *src, size_t wOffset, size_t hOffset, size_t width, size_t height, enum cudaMemcpyKind kind) +{ + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; +} + + +__host__ cudaError_t CUDARTAPI cudaMemcpy2DArrayToArray(struct cudaArray *dst, size_t wOffsetDst, size_t hOffsetDst, const struct cudaArray *src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, enum cudaMemcpyKind kind __dv(cudaMemcpyDeviceToDevice)) +{ + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; +} + + +__host__ cudaError_t CUDARTAPI cudaMemcpyToSymbol(const char *symbol, const void *src, size_t count, size_t offset __dv(0), enum cudaMemcpyKind kind __dv(cudaMemcpyHostToDevice)) +{ + //CUctx_st *context = GPGPUSim_Context(); + assert(kind == cudaMemcpyHostToDevice); + printf("GPGPU-Sim PTX: cudaMemcpyToSymbol: symbol = %p\n", symbol); + //stream_operation( const char *symbol, const void *src, size_t count, size_t offset ) + g_stream_manager->push( stream_operation(src,symbol,count,offset,0) ); + //gpgpu_ptx_sim_memcpy_symbol(symbol,src,count,offset,1,context->get_device()->get_gpgpu()); + return g_last_cudaError = cudaSuccess; +} + + +__host__ cudaError_t CUDARTAPI cudaMemcpyFromSymbol(void *dst, const char *symbol, size_t count, size_t offset __dv(0), enum cudaMemcpyKind kind __dv(cudaMemcpyDeviceToHost)) +{ + //CUctx_st *context = GPGPUSim_Context(); + assert(kind == cudaMemcpyDeviceToHost); + printf("GPGPU-Sim PTX: cudaMemcpyFromSymbol: symbol = %p\n", symbol); + g_stream_manager->push( stream_operation(symbol,dst,count,offset,0) ); + //gpgpu_ptx_sim_memcpy_symbol(symbol,dst,count,offset,0,context->get_device()->get_gpgpu()); + return g_last_cudaError = cudaSuccess; +} + + + +/******************************************************************************* + * * + * * + * * + *******************************************************************************/ + +__host__ cudaError_t CUDARTAPI cudaMemcpyAsync(void *dst, const void *src, size_t count, enum cudaMemcpyKind kind, cudaStream_t stream) +{ + struct CUstream_st *s = (struct CUstream_st *)stream; + switch( kind ) { + case cudaMemcpyHostToDevice: g_stream_manager->push( stream_operation(src,(size_t)dst,count,s) ); break; + case cudaMemcpyDeviceToHost: g_stream_manager->push( stream_operation((size_t)src,dst,count,s) ); break; + case cudaMemcpyDeviceToDevice: g_stream_manager->push( stream_operation((size_t)src,(size_t)dst,count,s) ); break; + default: + abort(); + } + return g_last_cudaError = cudaSuccess; +} + + +__host__ cudaError_t CUDARTAPI cudaMemcpyToArrayAsync(struct cudaArray *dst, size_t wOffset, size_t hOffset, const void *src, size_t count, enum cudaMemcpyKind kind, cudaStream_t stream) +{ + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; +} + + +__host__ cudaError_t CUDARTAPI cudaMemcpyFromArrayAsync(void *dst, const struct cudaArray *src, size_t wOffset, size_t hOffset, size_t count, enum cudaMemcpyKind kind, cudaStream_t stream) +{ + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; +} + + +__host__ cudaError_t CUDARTAPI cudaMemcpy2DAsync(void *dst, size_t dpitch, const void *src, size_t spitch, size_t width, size_t height, enum cudaMemcpyKind kind, cudaStream_t stream) +{ + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; +} + + +__host__ cudaError_t CUDARTAPI cudaMemcpy2DToArrayAsync(struct cudaArray *dst, size_t wOffset, size_t hOffset, const void *src, size_t spitch, size_t width, size_t height, enum cudaMemcpyKind kind, cudaStream_t stream) +{ + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; +} + + +__host__ cudaError_t CUDARTAPI cudaMemcpy2DFromArrayAsync(void *dst, size_t dpitch, const struct cudaArray *src, size_t wOffset, size_t hOffset, size_t width, size_t height, enum cudaMemcpyKind kind, cudaStream_t stream) +{ + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; +} + + + +/******************************************************************************* + * * + * * + * * + *******************************************************************************/ + +__host__ cudaError_t CUDARTAPI cudaMemset(void *mem, int c, size_t count) +{ + CUctx_st *context = GPGPUSim_Context(); + gpgpu_t *gpu = context->get_device()->get_gpgpu(); + gpu->gpu_memset((size_t)mem, c, count); + return g_last_cudaError = cudaSuccess; +} + +//memset operation is done but i think its not async? +__host__ cudaError_t CUDARTAPI cudaMemsetAsync(void *mem, int c, size_t count, cudaStream_t stream=0) +{ + printf("GPGPU-Sim PTX: WARNING: Asynchronous memset not supported (%s)\n", __my_func__); + CUctx_st *context = GPGPUSim_Context(); + gpgpu_t *gpu = context->get_device()->get_gpgpu(); + gpu->gpu_memset((size_t)mem, c, count); + return g_last_cudaError = cudaSuccess; +} + +__host__ cudaError_t CUDARTAPI cudaMemset2D(void *mem, size_t pitch, int c, size_t width, size_t height) +{ + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; +} + + + +/******************************************************************************* + * * + * * + * * + *******************************************************************************/ + +__host__ cudaError_t CUDARTAPI cudaGetSymbolAddress(void **devPtr, const char *symbol) +{ + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; +} + + +__host__ cudaError_t CUDARTAPI cudaGetSymbolSize(size_t *size, const char *symbol) +{ + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; +} + + + +/******************************************************************************* + * * + * * + * * + *******************************************************************************/ +__host__ cudaError_t CUDARTAPI cudaGetDeviceCount(int *count) +{ + _cuda_device_id *dev = GPGPUSim_Init(); + *count = dev->num_devices(); + return g_last_cudaError = cudaSuccess; +} + +__host__ cudaError_t CUDARTAPI cudaGetDeviceProperties(struct cudaDeviceProp *prop, int device) +{ + _cuda_device_id *dev = GPGPUSim_Init(); + if (device <= dev->num_devices() ) { + *prop= *dev->get_prop(); + return g_last_cudaError = cudaSuccess; + } else { + return g_last_cudaError = cudaErrorInvalidDevice; + } +} + +#if (CUDART_VERSION > 5000) +__host__ cudaError_t CUDARTAPI cudaDeviceGetAttribute(int *value, enum cudaDeviceAttr attr, int device) +{ + const struct cudaDeviceProp *prop; + _cuda_device_id *dev = GPGPUSim_Init(); + if (device <= dev->num_devices() ) { + prop = dev->get_prop(); + switch (attr) { + case 5: + *value= prop->maxGridSize[0]; + break; + case 6: + *value= prop->maxGridSize[1]; + break; + case 7: + *value= prop->maxGridSize[2]; + break; + case 10: + *value= prop->warpSize; + break; + case 12: + *value= prop->regsPerBlock; + break; + case 14: + *value= prop->textureAlignment ; + break; + case 16: + *value= prop->multiProcessorCount ; + break; + case 39: + *value= dev->get_gpgpu()->threads_per_core(); + break; + case 75: + *value= 8 ; + break; + case 76: + *value= 3 ; + break; + case 78: + *value= 0 ; //TODO: as of now, we dont support stream priorities. + break; + default: + printf("ERROR: implement the attribute numbered %d \n",attr); + abort(); + } + return g_last_cudaError = cudaSuccess; + } else { + return g_last_cudaError = cudaErrorInvalidDevice; + } +} +#endif + +__host__ cudaError_t CUDARTAPI cudaChooseDevice(int *device, const struct cudaDeviceProp *prop) +{ + _cuda_device_id *dev = GPGPUSim_Init(); + *device = dev->get_id(); + return g_last_cudaError = cudaSuccess; +} + +__host__ cudaError_t CUDARTAPI cudaSetDevice(int device) +{ + //set the active device to run cuda + if ( device <= GPGPUSim_Init()->num_devices() ) { + g_active_device = device; + return g_last_cudaError = cudaSuccess; + } else { + return g_last_cudaError = cudaErrorInvalidDevice; + } +} + +__host__ cudaError_t CUDARTAPI cudaGetDevice(int *device) +{ + *device = g_active_device; + return g_last_cudaError = cudaSuccess; +} + + +/******************************************************************************* + * * + * * + * * + *******************************************************************************/ + +__host__ cudaError_t CUDARTAPI cudaBindTexture(size_t *offset, + const struct textureReference *texref, + const void *devPtr, + const struct cudaChannelFormatDesc *desc, + size_t size __dv(UINT_MAX)) +{ + CUctx_st *context = GPGPUSim_Context(); + gpgpu_t *gpu = context->get_device()->get_gpgpu(); + printf("GPGPU-Sim PTX: in cudaBindTexture: sizeof(struct textureReference) = %zu\n", sizeof(struct textureReference)); + struct cudaArray *array; + array = (struct cudaArray*) malloc(sizeof(struct cudaArray)); + array->desc = *desc; + array->size = size; + array->width = size; + array->height = 1; + array->dimensions = 1; + array->devPtr = (void*)devPtr; + array->devPtr32 = (int)(long long)devPtr; + offset = 0; + printf("GPGPU-Sim PTX: size = %zu\n", size); + printf("GPGPU-Sim PTX: texref = %p, array = %p\n", texref, array); + printf("GPGPU-Sim PTX: devPtr32 = %x\n", array->devPtr32); + printf("GPGPU-Sim PTX: Name corresponding to textureReference: %s\n", gpu->gpgpu_ptx_sim_findNamefromTexture(texref)); + printf("GPGPU-Sim PTX: ChannelFormatDesc: x=%d, y=%d, z=%d, w=%d\n", desc->x, desc->y, desc->z, desc->w); + printf("GPGPU-Sim PTX: Texture Normalized? = %d\n", texref->normalized); + gpu->gpgpu_ptx_sim_bindTextureToArray(texref, array); + devPtr = (void*)(long long)array->devPtr32; + printf("GPGPU-Sim PTX: devPtr = %p\n", devPtr); + return g_last_cudaError = cudaSuccess; +} + + +__host__ cudaError_t CUDARTAPI cudaBindTextureToArray(const struct textureReference *texref, const struct cudaArray *array, const struct cudaChannelFormatDesc *desc) +{ + CUctx_st *context = GPGPUSim_Context(); + gpgpu_t *gpu = context->get_device()->get_gpgpu(); + printf("GPGPU-Sim PTX: in cudaBindTextureToArray: %p %p\n", texref, array); + printf("GPGPU-Sim PTX: devPtr32 = %x\n", array->devPtr32); + printf("GPGPU-Sim PTX: Name corresponding to textureReference: %s\n", gpu->gpgpu_ptx_sim_findNamefromTexture(texref)); + printf("GPGPU-Sim PTX: Texture Normalized? = %d\n", texref->normalized); + gpu->gpgpu_ptx_sim_bindTextureToArray(texref, array); + return g_last_cudaError = cudaSuccess; +} + +__host__ cudaError_t CUDARTAPI cudaUnbindTexture(const struct textureReference *texref) +{ + return g_last_cudaError = cudaSuccess; +} + +__host__ cudaError_t CUDARTAPI cudaGetTextureAlignmentOffset(size_t *offset, const struct textureReference *texref) +{ + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; +} + +__host__ cudaError_t CUDARTAPI cudaGetTextureReference(const struct textureReference **texref, const char *symbol) +{ + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; +} + +__host__ cudaError_t CUDARTAPI cudaGetChannelDesc(struct cudaChannelFormatDesc *desc, const struct cudaArray *array) +{ + *desc = array->desc; + return g_last_cudaError = cudaSuccess; +} + + +__host__ struct cudaChannelFormatDesc CUDARTAPI cudaCreateChannelDesc(int x, int y, int z, int w, enum cudaChannelFormatKind f) +{ + struct cudaChannelFormatDesc dummy; + dummy.x = x; + dummy.y = y; + dummy.z = z; + dummy.w = w; + dummy.f = f; + return dummy; +} + +__host__ cudaError_t CUDARTAPI cudaGetLastError(void) +{ + return g_last_cudaError; +} + +__host__ const char* CUDARTAPI cudaGetErrorString(cudaError_t error) +{ + if( g_last_cudaError == cudaSuccess ) + return "no error"; + char buf[1024]; + snprintf(buf,1024,"<>", g_last_cudaError); + return strdup(buf); +} + +__host__ cudaError_t CUDARTAPI cudaConfigureCall(dim3 gridDim, dim3 blockDim, size_t sharedMem, cudaStream_t stream) +{ + struct CUstream_st *s = (struct CUstream_st *)stream; + g_cuda_launch_stack.push_back( kernel_config(gridDim,blockDim,sharedMem,s) ); + return g_last_cudaError = cudaSuccess; +} + +__host__ cudaError_t CUDARTAPI cudaSetupArgument(const void *arg, size_t size, size_t offset) +{ + gpgpusim_ptx_assert( !g_cuda_launch_stack.empty(), "empty launch stack" ); + kernel_config &config = g_cuda_launch_stack.back(); + config.set_arg(arg,size,offset); + + return g_last_cudaError = cudaSuccess; +} + + +__host__ cudaError_t CUDARTAPI cudaLaunch( const char *hostFun ) +{ + CUctx_st* context = GPGPUSim_Context(); + char *mode = getenv("PTX_SIM_MODE_FUNC"); + if( mode ) + sscanf(mode,"%u", &g_ptx_sim_mode); + gpgpusim_ptx_assert( !g_cuda_launch_stack.empty(), "empty launch stack" ); + kernel_config config = g_cuda_launch_stack.back(); + struct CUstream_st *stream = config.get_stream(); + printf("\nGPGPU-Sim PTX: cudaLaunch for 0x%p (mode=%s) on stream %u\n", hostFun, + g_ptx_sim_mode?"functional simulation":"performance simulation", stream?stream->get_uid():0 ); + kernel_info_t *grid = gpgpu_cuda_ptx_sim_init_grid(hostFun,config.get_args(),config.grid_dim(),config.block_dim(),context); + //do dynamic PDOM analysis for performance simulation scenario + std::string kname = grid->name(); + function_info *kernel_func_info = grid->entry(); + if (kernel_func_info->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", kname.c_str() ); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", kname.c_str() ); + kernel_func_info->do_pdom(); + kernel_func_info->set_pdom(); + } + dim3 gridDim = config.grid_dim(); + dim3 blockDim = config.block_dim(); + printf("GPGPU-Sim PTX: pushing kernel \'%s\' to stream %u, gridDim= (%u,%u,%u) blockDim = (%u,%u,%u) \n", + kname.c_str(), stream?stream->get_uid():0, gridDim.x,gridDim.y,gridDim.z,blockDim.x,blockDim.y,blockDim.z ); + stream_operation op(grid,g_ptx_sim_mode,stream); + g_stream_manager->push(op); + g_cuda_launch_stack.pop_back(); + return g_last_cudaError = cudaSuccess; +} + +/******************************************************************************* + * * + * * + * * + *******************************************************************************/ + +__host__ cudaError_t CUDARTAPI cudaStreamCreate(cudaStream_t *stream) +{ + printf("GPGPU-Sim PTX: cudaStreamCreate\n"); +#if (CUDART_VERSION >= 3000) + *stream = new struct CUstream_st(); + g_stream_manager->add_stream(*stream); +#else + *stream = 0; + printf("GPGPU-Sim PTX: WARNING: Asynchronous kernel execution not supported (%s)\n", __my_func__); +#endif + return g_last_cudaError = cudaSuccess; +} + +//TODO: introduce priorities +__host__ cudaError_t CUDARTAPI cudaStreamCreateWithPriority(cudaStream_t *stream, unsigned int flags, int priority) { + return cudaStreamCreate(stream); +} + +__host__ cudaError_t CUDARTAPI cudaDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) { + return cudaSuccess; +} + +__host__ __device__ cudaError_t CUDARTAPI cudaStreamCreateWithFlags(cudaStream_t *pStream, unsigned int flags) { + return cudaStreamCreate(pStream); +} + +__host__ cudaError_t CUDARTAPI cudaStreamDestroy(cudaStream_t stream) +{ +#if (CUDART_VERSION >= 3000) + g_stream_manager->destroy_stream(stream); +#endif + return g_last_cudaError = cudaSuccess; +} + +__host__ cudaError_t CUDARTAPI cudaStreamSynchronize(cudaStream_t stream) +{ +#if (CUDART_VERSION >= 3000) + if( stream == NULL ) + synchronize(); + return g_last_cudaError = cudaSuccess; + stream->synchronize(); +#else + printf("GPGPU-Sim PTX: WARNING: Asynchronous kernel execution not supported (%s)\n", __my_func__); +#endif + return g_last_cudaError = cudaSuccess; +} + +__host__ cudaError_t CUDARTAPI cudaStreamQuery(cudaStream_t stream) +{ +#if (CUDART_VERSION >= 3000) + if( stream == NULL ) + return g_last_cudaError = cudaErrorInvalidResourceHandle; + return g_last_cudaError = stream->empty()?cudaSuccess:cudaErrorNotReady; +#else + printf("GPGPU-Sim PTX: WARNING: Asynchronous kernel execution not supported (%s)\n", __my_func__); + return g_last_cudaError = cudaSuccess; // it is always success because all cuda calls are synchronous +#endif +} + +/******************************************************************************* + * * + * * + * * + *******************************************************************************/ + +__host__ cudaError_t CUDARTAPI cudaEventCreate(cudaEvent_t *event) +{ + CUevent_st *e = new CUevent_st(false); + g_timer_events[e->get_uid()] = e; +#if CUDART_VERSION >= 3000 + *event = e; +#else + *event = e->get_uid(); +#endif + return g_last_cudaError = cudaSuccess; +} + +CUevent_st *get_event(cudaEvent_t event) +{ + unsigned event_uid; +#if CUDART_VERSION >= 3000 + event_uid = event->get_uid(); +#else + event_uid = event; +#endif + event_tracker_t::iterator e = g_timer_events.find(event_uid); + if( e == g_timer_events.end() ) + return NULL; + return e->second; +} + +__host__ cudaError_t CUDARTAPI cudaEventRecord(cudaEvent_t event, cudaStream_t stream) +{ + CUevent_st *e = get_event(event); + if( !e ) return g_last_cudaError = cudaErrorUnknown; + struct CUstream_st *s = (struct CUstream_st *)stream; + stream_operation op(e,s); + g_stream_manager->push(op); + return g_last_cudaError = cudaSuccess; +} + +__host__ cudaError_t CUDARTAPI cudaEventQuery(cudaEvent_t event) +{ + CUevent_st *e = get_event(event); + if( e == NULL ) { + return g_last_cudaError = cudaErrorInvalidValue; + } else if( e->done() ) { + return g_last_cudaError = cudaSuccess; + } else { + return g_last_cudaError = cudaErrorNotReady; + } +} + +__host__ cudaError_t CUDARTAPI cudaEventSynchronize(cudaEvent_t event) +{ + printf("GPGPU-Sim API: cudaEventSynchronize ** waiting for event\n"); + fflush(stdout); + CUevent_st *e = (CUevent_st*) event; + while( !e->done() ) + ; + printf("GPGPU-Sim API: cudaEventSynchronize ** event detected\n"); + fflush(stdout); + return g_last_cudaError = cudaSuccess; +} + +__host__ cudaError_t CUDARTAPI cudaEventDestroy(cudaEvent_t event) +{ + CUevent_st *e = get_event(event); + unsigned event_uid = e->get_uid(); + event_tracker_t::iterator pe = g_timer_events.find(event_uid); + if( pe == g_timer_events.end() ) + return g_last_cudaError = cudaErrorInvalidValue; + g_timer_events.erase(pe); + return g_last_cudaError = cudaSuccess; +} + + +__host__ cudaError_t CUDARTAPI cudaEventElapsedTime(float *ms, cudaEvent_t start, cudaEvent_t end) +{ + time_t elapsed_time; + CUevent_st *s = get_event(start); + CUevent_st *e = get_event(end); + if( s==NULL || e==NULL ) + return g_last_cudaError = cudaErrorUnknown; + elapsed_time = e->clock() - s->clock(); + *ms = 1000*elapsed_time; + return g_last_cudaError = cudaSuccess; +} + + + +/******************************************************************************* + * * + * * + * * + *******************************************************************************/ + +__host__ cudaError_t CUDARTAPI cudaThreadExit(void) +{ + exit_simulation(); + return g_last_cudaError = cudaSuccess; +} + +__host__ cudaError_t CUDARTAPI cudaThreadSynchronize(void) +{ + //Called on host side + synchronize(); + return g_last_cudaError = cudaSuccess; +}; + +int CUDARTAPI __cudaSynchronizeThreads(void**, void*) +{ + return cudaThreadExit(); +} + + + +/******************************************************************************* + * * + * * + * * + *******************************************************************************/ + +#if (CUDART_VERSION >= 3010) + +typedef struct CUuuid_st { /**< CUDA definition of UUID */ + char bytes[16]; +} CUuuid; + +/** + * CUDA UUID types + */ +// typedef __device_builtin__ struct CUuuid_st cudaUUID_t; + +__host__ cudaError_t CUDARTAPI cudaGetExportTable(const void **ppExportTable, const cudaUUID_t *pExportTableId) +{ + printf("cudaGetExportTable: UUID = "); + for (int s = 0; s < 16; s++) { + printf("%#2x ", (unsigned char) (pExportTableId->bytes[s])); + } + printf("\n"); + return g_last_cudaError = cudaSuccess; +} + +#endif + + +/******************************************************************************* + * * + * * + * * + *******************************************************************************/ + +//#include "../../cuobjdump_to_ptxplus/cuobjdump_parser.h" + +enum cuobjdumpSectionType { + PTXSECTION=0, + ELFSECTION +}; + + +class cuobjdumpSection { +public: + //Constructor + cuobjdumpSection() { + arch = 0; + identifier = ""; + } + virtual ~cuobjdumpSection() {} + unsigned getArch() {return arch;} + void setArch(unsigned a) {arch = a;} + std::string getIdentifier() {return identifier;} + void setIdentifier(std::string i) {identifier = i;} + virtual void print(){std::cout << "cuobjdump Section: unknown type" << std::endl;} +private: + unsigned arch; + std::string identifier; +}; + +class cuobjdumpELFSection : public cuobjdumpSection +{ +public: + cuobjdumpELFSection() {} + virtual ~cuobjdumpELFSection() { + elffilename = ""; + sassfilename = ""; + } + std::string getELFfilename() {return elffilename;} + void setELFfilename(std::string f) {elffilename = f;} + std::string getSASSfilename() {return sassfilename;} + void setSASSfilename(std::string f) {sassfilename = f;} + virtual void print() { + std::cout << "ELF Section:" << std::endl; + std::cout << "arch: sm_" << getArch() << std::endl; + std::cout << "identifier: " << getIdentifier() << std::endl; + std::cout << "elf filename: " << getELFfilename() << std::endl; + std::cout << "sass filename: " << getSASSfilename() << std::endl; + std::cout << std::endl; + } +private: + std::string elffilename; + std::string sassfilename; +}; + +class cuobjdumpPTXSection : public cuobjdumpSection +{ +public: + cuobjdumpPTXSection(){ + ptxfilename = ""; + } + std::string getPTXfilename() {return ptxfilename;} + void setPTXfilename(std::string f) {ptxfilename = f;} + virtual void print() { + std::cout << "PTX Section:" << std::endl; + std::cout << "arch: sm_" << getArch() << std::endl; + std::cout << "identifier: " << getIdentifier() << std::endl; + std::cout << "ptx filename: " << getPTXfilename() << std::endl; + std::cout << std::endl; + } +private: + std::string ptxfilename; +}; + +std::list cuobjdumpSectionList; +std::list libSectionList; + +// sectiontype: 0 for ptx, 1 for elf +void addCuobjdumpSection(int sectiontype){ + if (sectiontype) + cuobjdumpSectionList.push_front(new cuobjdumpELFSection()); + else + cuobjdumpSectionList.push_front(new cuobjdumpPTXSection()); + printf("## Adding new section %s\n", sectiontype?"ELF":"PTX"); +} + +void setCuobjdumparch(const char* arch){ + unsigned archnum; + sscanf(arch, "sm_%u", &archnum); + assert (archnum && "cannot have sm_0"); + printf("Adding arch: %s\n", arch); + cuobjdumpSectionList.front()->setArch(archnum); +} + +void setCuobjdumpidentifier(const char* identifier){ + printf("Adding identifier: %s\n", identifier); + cuobjdumpSectionList.front()->setIdentifier(identifier); +} + +void setCuobjdumpptxfilename(const char* filename){ + printf("Adding ptx filename: %s\n", filename); + cuobjdumpSection* x = cuobjdumpSectionList.front(); + if (dynamic_cast(x) == NULL){ + assert (0 && "You shouldn't be trying to add a ptxfilename to an elf section"); + } + (dynamic_cast(x))->setPTXfilename(filename); +} + +void setCuobjdumpelffilename(const char* filename){ + if (dynamic_cast(cuobjdumpSectionList.front()) == NULL){ + assert (0 && "You shouldn't be trying to add a elffilename to an ptx section"); + } + (dynamic_cast(cuobjdumpSectionList.front()))->setELFfilename(filename); +} + +void setCuobjdumpsassfilename(const char* filename){ + if (dynamic_cast(cuobjdumpSectionList.front()) == NULL){ + assert (0 && "You shouldn't be trying to add a sassfilename to an ptx section"); + } + (dynamic_cast(cuobjdumpSectionList.front()))->setSASSfilename(filename); +} +extern int cuobjdump_parse(); +extern FILE *cuobjdump_in; + +//! Return the executable file of the process containing the PTX/SASS code +//! +//! This Function returns the executable file ran by the process. This +//! executable is supposed to contain the PTX/SASS code. It provides workaround +//! for processes running on valgrind by dereferencing /proc//exe within the +//! GPGPU-Sim process before calling cuobjdump to extract PTX/SASS. This is +//! needed because valgrind uses x86 emulation to detect memory leak. Other +//! processes (e.g. cuobjdump) reading /proc//exe will see the emulator +//! executable instead of the application binary. +//! +std::string get_app_binary(){ + char self_exe_path[1025]; +#ifdef __APPLE__ + uint32_t size = sizeof(self_exe_path); + if( _NSGetExecutablePath(self_exe_path,&size) != 0 ) { + printf("GPGPU-Sim ** ERROR: _NSGetExecutablePath input buffer too small\n"); + exit(1); + } +#else + std::stringstream exec_link; + exec_link << "/proc/self/exe"; + + ssize_t path_length = readlink(exec_link.str().c_str(), self_exe_path, 1024); + assert(path_length != -1); + self_exe_path[path_length] = '\0'; +#endif + + printf("self exe links to: %s\n", self_exe_path); + return self_exe_path; +} + +//above func gives abs path whereas this give just the name of application. +char* get_app_binary_name(std::string abs_path){ + char *self_exe_path; +#ifdef __APPLE__ + //TODO: get apple device and check the result. + printf("WARNING: not tested for Apple-mac devices \n"); + abort(); +#else + char* buf = strdup(abs_path.c_str()); + char *token = strtok(buf, "/"); + while(token !=NULL){ + self_exe_path = token; + token = strtok(NULL,"/"); + } +#endif + self_exe_path = strtok(self_exe_path, "."); + printf("self exe links to: %s\n", self_exe_path); + return self_exe_path; +} + +//! Call cuobjdump to extract everything (-elf -sass -ptx) +/*! + * This Function extract the whole PTX (for all the files) using cuobjdump + * to _cuobjdump_complete_output_XXXXXX then runs a parser to chop it up with each binary in + * its own file + * It is also responsible for extracting the libraries linked to the binary if the option is + * enabled + * */ +void extract_code_using_cuobjdump(){ + CUctx_st *context = GPGPUSim_Context(); + unsigned forced_max_capability = context->get_device()->get_gpgpu()->get_config().get_forced_max_capability(); + + //prevent the dumping by cuobjdump everytime we execute the code! + const char *override_cuobjdump = getenv("CUOBJDUMP_SIM_FILE"); + char command[1000], ptx_file[1000]; + std::string app_binary = get_app_binary(); + //Running cuobjdump using dynamic link to current process + snprintf(command,1000,"md5sum %s ", app_binary.c_str()); + printf("Running md5sum using \"%s\"\n", command); + system(command); + // Running cuobjdump using dynamic link to current process + // Needs the option '-all' to extract PTX from CDP-enabled binary + extern bool g_cdp_enabled; + + //dump ptx for all individial ptx files into sepearte files which is later used by ptxas. + int result=0; +#if (CUDART_VERSION >= 6000) + char fname2[1024]; + snprintf(fname2,1024,"_cuobjdump_list_ptx_XXXXXX"); + int fd2=mkstemp(fname2); + close(fd2); + snprintf(command,1000,"$CUDA_INSTALL_PATH/bin/cuobjdump -lptx -arch=sm_%u %s > %s", forced_max_capability, app_binary.c_str(), fname2); + result = system(command); + if( result != 0 ) { + printf("WARNING: Failed to execute cuobjdump to get list of ptx files \n"); + exit(0); + } else { + /* + as we got list of ptx files, we need to extract one by one into seperate files so that ptxas can understand it. + In this way, the duplicate definitions in a single embedded file can be prevented. + No of lines in the file is equal to no of ptx fileis available. + */ + FILE *fp = fopen(fname2,"r"); + if (fp==NULL) { + printf("WARNING: cuobjdump file error! Could not open file %s \n", fname2); + exit(0); + } else { + for (char c = getc(fp); c != EOF; c = getc(fp)) + if (c == '\n') + no_of_ptx = no_of_ptx + 1; + fclose(fp); + } + if(no_of_ptx==0){ + printf("WARNING: Number of ptx in the executable file are 0. One of the reasons might be\n"); + printf("\t1. CDP is enabled\n"); + printf("\t2. cuobjdump -lptx doesnt recognize sm_%u\n",forced_max_capability); + printf("\t3. the application was not compiled iwth nvcc flag sm_%u\n",forced_max_capability); + } + } + if(!g_cdp_enabled) { + //based on the list above, dump ptx files individually. Format of dumped ptx file is prog_name.unique_no.sm_<>.ptx + for (int index=1; index<= no_of_ptx; index++){ + snprintf(ptx_file, 1000, "%s.%d.sm_%u.ptx", get_app_binary_name(app_binary), index, forced_max_capability); + printf("Extracting specific PTX file named %s \n",ptx_file); + snprintf(command,1000,"$CUDA_INSTALL_PATH/bin/cuobjdump -arch=sm_%u -xptx %s %s", forced_max_capability, ptx_file, app_binary.c_str()); + if (system(command)!=0) { + printf("ERROR: command: %s failed \n",command); + exit(0); + } + } + } +#endif + //TODO: redundant to dump twice. how can it be prevented? + //dump only for specific arch + char fname[1024]; + if ((override_cuobjdump == NULL) || (strlen(override_cuobjdump)==0)) { + snprintf(fname,1024,"_cuobjdump_complete_output_XXXXXX"); + int fd=mkstemp(fname); + close(fd); + if(!g_cdp_enabled) + snprintf(command,1000,"$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass %s > %s", app_binary.c_str(), fname); + else + snprintf(command,1000,"$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass -all %s > %s", app_binary.c_str(), fname); + bool parse_output = true; + result = system(command); + if(result) { + if (context->get_device()->get_gpgpu()->get_config().experimental_lib_support() && (result == 65280)) { + // Some CUDA application may exclusively use kernels provided by CUDA + // libraries (e.g. CUBLAS). Skipping cuobjdump extraction from the + // executable for this case. + // 65280 is the return code from cuobjdump denoting the specific error (tested on CUDA 4.0/4.1/4.2) + printf("WARNING: Failed to execute: %s\n", command); + printf(" Executable binary does not contain any GPU kernel.\n"); + parse_output = false; + } else { + printf("ERROR: Failed to execute: %s\n", command); + exit(1); + } + } + + if (parse_output) { + printf("Parsing file %s\n", fname); + cuobjdump_in = fopen(fname, "r"); + + cuobjdump_parse(); + fclose(cuobjdump_in); + printf("Done parsing!!!\n"); + } else { + printf("Parsing skipped for %s\n", fname); + } + + if (context->get_device()->get_gpgpu()->get_config().experimental_lib_support()){ + //Experimental library support + //Currently only for cufft + + std::stringstream cmd; + cmd << "ldd " << app_binary << " | grep $CUDA_INSTALL_PATH | awk \'{print $3}\' > _tempfile_.txt"; + int result = system(cmd.str().c_str()); + if(result){ + std::cout << "Failed to execute: " << cmd.str() << std::endl; + exit(1); + } + std::ifstream libsf; + libsf.open("_tempfile_.txt"); + if(!libsf.is_open()) { + std::cout << "Failed to open: _tempfile_.txt" << std::endl; + exit(1); + } + + //Save the original section list + std::list tmpsl = cuobjdumpSectionList; + cuobjdumpSectionList.clear(); + + std::string line; + std::getline(libsf, line); + std::cout << "DOING: " << line << std::endl; + int cnt=1; + while(libsf.good()){ + std::stringstream libcodfn; + libcodfn << "_cuobjdump_complete_lib_" << cnt << "_"; + cmd.str(""); //resetting + cmd << "$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass "; + cmd << line; + cmd << " > "; + cmd << libcodfn.str(); + std::cout << "Running cuobjdump on " << line << std::endl; + std::cout << "Using command: " << cmd.str() << std::endl; + result = system(cmd.str().c_str()); + if(result) {printf("ERROR: Failed to execute: %s\n", command); exit(1);} + std::cout << "Done" << std::endl; + + std::cout << "Trying to parse " << libcodfn.str() << std::endl; + cuobjdump_in = fopen(libcodfn.str().c_str(), "r"); + cuobjdump_parse(); + fclose(cuobjdump_in); + std::getline(libsf, line); + } + libSectionList = cuobjdumpSectionList; + + //Restore the original section list + cuobjdumpSectionList = tmpsl; + } + } else { + printf("GPGPU-Sim PTX: overriding cuobjdump with '%s' (CUOBJDUMP_SIM_FILE is set)\n", override_cuobjdump); + snprintf(fname,1024, "%s",override_cuobjdump); + } +} + +//! Read file into char* +//TODO: convert this to C++ streams, will be way cleaner +char* readfile (const std::string filename){ + assert (filename != ""); + FILE* fp = fopen(filename.c_str(),"r"); + if (!fp) { + std::cout << "ERROR: Could not open file %s for reading\n" << filename << std::endl; + assert (0); + } + // finding size of the file + int filesize= 0; + fseek (fp , 0 , SEEK_END); + + filesize = ftell (fp); + fseek (fp, 0, SEEK_SET); + // allocate and copy the entire ptx + char* ret = (char*)malloc((filesize +1)* sizeof(char)); + fread(ret,1,filesize,fp); + ret[filesize]='\0'; + fclose(fp); + return ret; +} + +//! Function that helps debugging +void printSectionList(std::list sl) { + std::list::iterator iter; + for ( iter = sl.begin(); + iter != sl.end(); + iter++ + ){ + (*iter)->print(); + } +} + +//! Remove unecessary sm versions from the section list +std::list pruneSectionList(std::list cuobjdumpSectionList, CUctx_st *context) { + unsigned forced_max_capability = context->get_device()->get_gpgpu()->get_config().get_forced_max_capability(); + + //For ptxplus, force the max capability to 19 if it's higher or unspecified(0) + if (context->get_device()->get_gpgpu()->get_config().convert_to_ptxplus()){ + if ( (forced_max_capability == 0) || + (forced_max_capability >= 20)){ + printf("GPGPU-Sim: WARNING: Capability >= 20 are not supported in PTXPlus\n\tSetting forced_max_capability to 19\n"); + forced_max_capability = 19; + } + } + + std::list prunedList; + + //Find the highest capability (that is lower than the forced maximum) for each cubin file + //and set it in cuobjdumpSectionMap. Do this only for ptx sections + std::map cuobjdumpSectionMap; + int min_ptx_capability_found=0; + for ( std::list::iterator iter = cuobjdumpSectionList.begin(); + iter != cuobjdumpSectionList.end(); + iter++){ + unsigned capability = (*iter)->getArch(); + if(dynamic_cast(*iter) != NULL){ + if(capabilitygetIdentifier())==cuobjdumpSectionMap.end()) + || (cuobjdumpSectionMap[(*iter)->getIdentifier()] < capability)) + cuobjdumpSectionMap[(*iter)->getIdentifier()] = capability; + } + } + } + + //Throw away the sections with the lower capabilites and push those with the highest in + //the pruned list + for ( std::list::iterator iter = cuobjdumpSectionList.begin(); + iter != cuobjdumpSectionList.end(); + iter++){ + unsigned capability = (*iter)->getArch(); + if(capability == cuobjdumpSectionMap[(*iter)->getIdentifier()]){ + prunedList.push_back(*iter); + } else { + delete *iter; + } + } + if(prunedList.empty()){ + printf("Error: No PTX sections found with sm capability that is lower than current forced maximum capability \n minimum ptx capability found = %u, maximum forced ptx capability = %u \n User might want to change either the forced maximum capability from gpgpusim configuration or update the compilation to generate the required PTX version\n",min_ptx_capability_found,forced_max_capability); + abort(); + } + return prunedList; +} + +//! Merge all PTX sections that have a specific identifier into one file +std::list mergeMatchingSections(std::list cuobjdumpSectionList, std::string identifier){ + const char *ptxcode = ""; + std::list::iterator old_iter; + cuobjdumpPTXSection* old_ptxsection = NULL; + cuobjdumpPTXSection* ptxsection; + std::list mergedList; + + for ( std::list::iterator iter = cuobjdumpSectionList.begin(); + iter != cuobjdumpSectionList.end(); + iter++){ + if((ptxsection=dynamic_cast(*iter)) != NULL && + strcmp(ptxsection->getIdentifier().c_str(), identifier.c_str()) == 0){ + // Read and remove the last PTX section + if (old_ptxsection != NULL) { + ptxcode = readfile(old_ptxsection->getPTXfilename()); + // remove ptx file? + delete *old_iter; + } + + // Append all the PTX from the last PTX section into the current PTX section + // Add 50 to ptxcode to ignore the information regarding version/target/address_size + if (strlen(ptxcode) >= 50) { + FILE *ptxfile = fopen((ptxsection->getPTXfilename()).c_str(), "a"); + fprintf(ptxfile, "%s", ptxcode + 50); + fclose(ptxfile); + } + + old_iter = iter; + old_ptxsection = ptxsection; + } + // Store all non-PTX sections and PTX sections with non-matching identifiers + else { + mergedList.push_back(*iter); + } + } + + // Store the final PTX section + mergedList.push_back(*old_iter); + + return mergedList; +} + +//! Merge any PTX sections with matching identifiers +std::list mergeSections(std::list cuobjdumpSectionList){ + std::vector identifier; + cuobjdumpPTXSection* ptxsection; + + // Add all identifiers present in PTX sections to a vector + for ( std::list::iterator iter = cuobjdumpSectionList.begin(); + iter != cuobjdumpSectionList.end(); + iter++){ + if((ptxsection=dynamic_cast(*iter)) != NULL){ + std::string current_id = ptxsection->getIdentifier(); + + // If we haven't yet seen a given identifier, add it to the vector + if (std::find(identifier.begin(), identifier.end(), current_id) == identifier.end()) { + identifier.push_back(current_id); + } + } + } + + // Call mergeMatchingSections on all identifiers in the vector + for ( std::vector::iterator iter = identifier.begin(); + iter != identifier.end(); + iter++) { + cuobjdumpSectionList = mergeMatchingSections(cuobjdumpSectionList, *iter); + } + + return cuobjdumpSectionList; +} + + +//! Within the section list, find the ELF section corresponding to a given identifier +cuobjdumpELFSection* findELFSectionInList(std::list sectionlist, const std::string identifier){ + + std::list::iterator iter; + for ( iter = sectionlist.begin(); + iter != sectionlist.end(); + iter++ + ){ + cuobjdumpELFSection* elfsection; + if((elfsection=dynamic_cast(*iter)) != NULL){ + if(elfsection->getIdentifier() == identifier) + return elfsection; + } + } + return NULL; +} + +//! Find an ELF section in all the known lists +cuobjdumpELFSection* findELFSection(const std::string identifier){ + cuobjdumpELFSection* sec = findELFSectionInList(cuobjdumpSectionList, identifier); + if (sec!=NULL)return sec; + sec = findELFSectionInList(libSectionList, identifier); + if (sec!=NULL)return sec; + std::cout << "Could not find " << identifier << std::endl; + assert(0 && "Could not find the required ELF section"); + return NULL; +} + +//! Within the section list, find the PTX section corresponding to a given identifier +cuobjdumpPTXSection* findPTXSectionInList(std::list sectionlist, const std::string identifier){ + std::list::iterator iter; + for ( iter = sectionlist.begin(); + iter != sectionlist.end(); + iter++ + ){ + cuobjdumpPTXSection* ptxsection; + if((ptxsection=dynamic_cast(*iter)) != NULL){ + if(ptxsection->getIdentifier() == identifier) + return ptxsection; + else { + extern bool g_cdp_enabled; + if(g_cdp_enabled) { + printf("Warning: __cudaRegisterFatBinary needs %s, but find PTX section with %s\n", + identifier.c_str(), ptxsection->getIdentifier().c_str()); + return ptxsection; + } + } + } + } + return NULL; +} + +//! Find an PTX section in all the known lists +cuobjdumpPTXSection* findPTXSection(const std::string identifier){ + cuobjdumpPTXSection* sec = findPTXSectionInList(cuobjdumpSectionList, identifier); + if (sec!=NULL)return sec; + sec = findPTXSectionInList(libSectionList, identifier); + if (sec!=NULL)return sec; + std::cout << "Could not find " << identifier << std::endl; + assert(0 && "Could not find the required PTX section"); + return NULL; +} + + + +//! Extract the code using cuobjdump and remove unnecessary sections +void cuobjdumpInit(){ + CUctx_st *context = GPGPUSim_Context(); + extract_code_using_cuobjdump(); //extract all the output of cuobjdump to _cuobjdump_*.* + const char* pre_load = getenv("CUOBJDUMP_SIM_FILE"); + if (pre_load ==NULL || strlen(pre_load)==0){ + cuobjdumpSectionList = pruneSectionList(cuobjdumpSectionList, context); + cuobjdumpSectionList = mergeSections(cuobjdumpSectionList); + } +} + +std::map fatbinmap; +std::mapfatbin_registered; +std::map name_symtab; + +//! Keep track of the association between filename and cubin handle +void cuobjdumpRegisterFatBinary(unsigned int handle, const char* filename){ + fatbinmap[handle] = filename; +} + +//! Either submit PTX for simulation or convert SASS to PTXPlus and submit it +void cuobjdumpParseBinary(unsigned int handle){ + + if(fatbin_registered[handle]) return; + fatbin_registered[handle] = true; + CUctx_st *context = GPGPUSim_Context(); + std::string fname = fatbinmap[handle]; + + if (name_symtab.find(fname) != name_symtab.end()) { + symbol_table *symtab = name_symtab[fname]; + context->add_binary(symtab, handle); + return; + } + + unsigned max_capability = 0; + for ( std::list::iterator iter = cuobjdumpSectionList.begin(); + iter != cuobjdumpSectionList.end(); + iter++){ + unsigned capability = (*iter)->getArch(); + if (capability > max_capability) max_capability = capability; + } + if (max_capability > 20) printf("WARNING: No guarantee that PTX will be parsed for SM version %u\n", max_capability); + + cuobjdumpPTXSection* ptx = NULL; + const char* pre_load = getenv("CUOBJDUMP_SIM_FILE"); + if(pre_load==NULL || strlen(pre_load)==0) + ptx = findPTXSection(fname); + symbol_table *symtab; + char *ptxcode; + const char *override_ptx_name = getenv("PTX_SIM_KERNELFILE"); + if (override_ptx_name == NULL or getenv("PTX_SIM_USE_PTX_FILE") == NULL or strlen(getenv("PTX_SIM_USE_PTX_FILE"))==0) { + ptxcode = readfile(ptx->getPTXfilename()); + } else { + printf("GPGPU-Sim PTX: overriding embedded ptx with '%s' (PTX_SIM_USE_PTX_FILE is set)\n", override_ptx_name); + ptxcode = readfile(override_ptx_name); + } + if(context->get_device()->get_gpgpu()->get_config().convert_to_ptxplus() ) { + cuobjdumpELFSection* elfsection = findELFSection(ptx->getIdentifier()); + assert (elfsection!= NULL); + char *ptxplus_str = gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus( + ptx->getPTXfilename(), + elfsection->getELFfilename(), + elfsection->getSASSfilename()); + symtab=gpgpu_ptx_sim_load_ptx_from_string(ptxplus_str, handle); + printf("Adding %s with cubin handle %u\n", ptx->getPTXfilename().c_str(), handle); + context->add_binary(symtab, handle); + gpgpu_ptxinfo_load_from_string( ptxcode, handle, max_capability ); + delete[] ptxplus_str; + } else { + symtab=gpgpu_ptx_sim_load_ptx_from_string(ptxcode, handle); + //if CUOBJDUMP_SIM_FILE is not set, ptx is NULL. So comment below. + //printf("Adding %s with cubin handle %u\n", ptx->getPTXfilename().c_str(), handle); + context->add_binary(symtab, handle); + gpgpu_ptxinfo_load_from_string( ptxcode, handle, max_capability ); + } + load_static_globals(symtab,STATIC_ALLOC_LIMIT,0xFFFFFFFF,context->get_device()->get_gpgpu()); + load_constants(symtab,STATIC_ALLOC_LIMIT,context->get_device()->get_gpgpu()); + name_symtab[fname] = symtab; + + //TODO: Remove temporarily files as per configurations +} + +void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin ) +{ +#if (CUDART_VERSION < 2010) + printf("GPGPU-Sim PTX: ERROR ** this version of GPGPU-Sim requires CUDA 2.1 or higher\n"); + exit(1); +#endif + CUctx_st *context = GPGPUSim_Context(); + static unsigned next_fat_bin_handle = 1; + if(context->get_device()->get_gpgpu()->get_config().use_cuobjdump()) { + // The following workaround has only been verified on 64-bit systems. + if (sizeof(void*) == 4) + printf("GPGPU-Sim PTX: FatBin file name extraction has not been tested on 32-bit system.\n"); + + #if (CUDART_VERSION <= 6000) + // FatBin handle from the .fatbin.c file (one of the intermediate files generated by NVCC) + typedef struct {int m; int v; const unsigned long long* d; char* f;} __fatDeviceText __attribute__ ((aligned (8))); + __fatDeviceText * fatDeviceText = (__fatDeviceText *) fatCubin; + + // Extract the source code file name that generate the given FatBin. + // - Obtains the pointer to the actual fatbin structure from the FatBin handle (fatCubin). + // - An integer inside the fatbin structure contains the relative offset to the source code file name. + // - This offset differs among different CUDA and GCC versions. + char * pfatbin = (char*) fatDeviceText->d; + int offset = *((int*)(pfatbin+48)); + char * filename = (pfatbin+16+offset); + #else + const char * filename = "default"; + #endif + // The extracted file name is associated with a fat_cubin_handle passed + // into cudaLaunch(). Inside cudaLaunch(), the associated file name is + // used to find the PTX/SASS section from cuobjdump, which contains the + // PTX/SASS code for the launched kernel function. + // This allows us to work around the fact that cuobjdump only outputs the + // file name associated with each section. + unsigned long long fat_cubin_handle = next_fat_bin_handle; + next_fat_bin_handle++; + printf("GPGPU-Sim PTX: __cudaRegisterFatBinary, fat_cubin_handle = %llu, filename=%s\n", fat_cubin_handle, filename); + /*! + * This function extracts all data from all files in first call + * then for next calls, only returns the appropriate number + */ + assert(fat_cubin_handle >= 1); + if (fat_cubin_handle==1) cuobjdumpInit(); + cuobjdumpRegisterFatBinary(fat_cubin_handle, filename); + + return (void**)fat_cubin_handle; + } +#if (CUDART_VERSION < 8000) + else { + static unsigned source_num=1; + unsigned long long fat_cubin_handle = next_fat_bin_handle++; + __cudaFatCudaBinary *info = (__cudaFatCudaBinary *)fatCubin; + assert( info->version >= 3 ); + unsigned num_ptx_versions=0; + unsigned max_capability=0; + unsigned selected_capability=0; + bool found=false; + unsigned forced_max_capability = context->get_device()->get_gpgpu()->get_config().get_forced_max_capability(); + if (!info->ptx){ + printf("ERROR: Cannot find ptx code in cubin file\n" + "\tIf you are using CUDA 4.0 or higher, please enable -gpgpu_ptx_use_cuobjdump or downgrade to CUDA 3.1\n"); + exit(1); + } + while( info->ptx[num_ptx_versions].gpuProfileName != NULL ) { + unsigned capability=0; + sscanf(info->ptx[num_ptx_versions].gpuProfileName,"compute_%u",&capability); + printf("GPGPU-Sim PTX: __cudaRegisterFatBinary found PTX versions for '%s', ", info->ident); + printf("capability = %s\n", info->ptx[num_ptx_versions].gpuProfileName ); + if( forced_max_capability ) { + if( capability > max_capability && capability <= forced_max_capability ) { + found = true; + max_capability=capability; + selected_capability = num_ptx_versions; + } + } else { + if( capability > max_capability ) { + found = true; + max_capability=capability; + selected_capability = num_ptx_versions; + } + } + num_ptx_versions++; + } + if( found ) { + printf("GPGPU-Sim PTX: Loading PTX for %s, capability = %s\n", + info->ident, info->ptx[selected_capability].gpuProfileName ); + symbol_table *symtab; + const char *ptx = info->ptx[selected_capability].ptx; + if(context->get_device()->get_gpgpu()->get_config().convert_to_ptxplus() ) { + printf("GPGPU-Sim PTX: ERROR ** PTXPlus is only supported through cuobjdump\n" + "\tEither enable cuobjdump or disable PTXPlus in your configuration file\n"); + exit(1); + } else { + symtab=gpgpu_ptx_sim_load_ptx_from_string(ptx,source_num); + context->add_binary(symtab,fat_cubin_handle); + gpgpu_ptxinfo_load_from_string( ptx, source_num, max_capability ); + } + source_num++; + load_static_globals(symtab,STATIC_ALLOC_LIMIT,0xFFFFFFFF,context->get_device()->get_gpgpu()); + load_constants(symtab,STATIC_ALLOC_LIMIT,context->get_device()->get_gpgpu()); + } else { + printf("GPGPU-Sim PTX: warning -- did not find an appropriate PTX in cubin\n"); + } + return (void**)fat_cubin_handle; + } +#else + else { + printf("ERROR ** __cudaRegisterFatBinary() needs to be updated\n"); + abort(); + } +#endif +} + +void __cudaUnregisterFatBinary(void **fatCubinHandle) +{ + ; +} + +cudaError_t cudaDeviceReset ( void ) { + // Should reset the simulated GPU + return g_last_cudaError = cudaSuccess; +} +cudaError_t CUDARTAPI cudaDeviceSynchronize(void){ + // I don't know what this should do + return g_last_cudaError = cudaSuccess; +} + + +void CUDARTAPI __cudaRegisterFunction( + void **fatCubinHandle, + const char *hostFun, + char *deviceFun, + const char *deviceName, + int thread_limit, + uint3 *tid, + uint3 *bid, + dim3 *bDim, + dim3 *gDim +) +{ + CUctx_st *context = GPGPUSim_Context(); + unsigned fat_cubin_handle = (unsigned)(unsigned long long)fatCubinHandle; + printf("GPGPU-Sim PTX: __cudaRegisterFunction %s : hostFun 0x%p, fat_cubin_handle = %u\n", + deviceFun, hostFun, fat_cubin_handle); + if(context->get_device()->get_gpgpu()->get_config().use_cuobjdump()) + cuobjdumpParseBinary(fat_cubin_handle); + context->register_function( fat_cubin_handle, hostFun, deviceFun ); +} + +extern void __cudaRegisterVar( + void **fatCubinHandle, + char *hostVar, //pointer to...something + char *deviceAddress, //name of variable + const char *deviceName, //name of variable (same as above) + int ext, + int size, + int constant, + int global ) +{ + printf("GPGPU-Sim PTX: __cudaRegisterVar: hostVar = %p; deviceAddress = %s; deviceName = %s\n", hostVar, deviceAddress, deviceName); + printf("GPGPU-Sim PTX: __cudaRegisterVar: Registering const memory space of %d bytes\n", size); + if(GPGPUSim_Context()->get_device()->get_gpgpu()->get_config().use_cuobjdump()) + cuobjdumpParseBinary((unsigned)(unsigned long long)fatCubinHandle); + fflush(stdout); + if ( constant && !global && !ext ) { + gpgpu_ptx_sim_register_const_variable(hostVar,deviceName,size); + } else if ( !constant && !global && !ext ) { + gpgpu_ptx_sim_register_global_variable(hostVar,deviceName,size); + } else cuda_not_implemented(__my_func__,__LINE__); +} + + +void __cudaRegisterShared( + void **fatCubinHandle, + void **devicePtr +) +{ + // we don't do anything here + printf("GPGPU-Sim PTX: __cudaRegisterShared\n" ); +} + +void CUDARTAPI __cudaRegisterSharedVar( + void **fatCubinHandle, + void **devicePtr, + size_t size, + size_t alignment, + int storage +) +{ + // we don't do anything here + printf("GPGPU-Sim PTX: __cudaRegisterSharedVar\n" ); +} + +void __cudaRegisterTexture( + void **fatCubinHandle, + const struct textureReference *hostVar, + const void **deviceAddress, + const char *deviceName, + int dim, + int norm, + int ext +) //passes in a newly created textureReference +{ + std::string devStr (deviceName); + #if (CUDART_VERSION > 4020) + if (devStr.size() > 2 && devStr.data()[0] == ':' && devStr.data()[1] == ':') + devStr = devStr.replace(0, 2, ""); + #endif + CUctx_st *context = GPGPUSim_Context(); + gpgpu_t *gpu = context->get_device()->get_gpgpu(); + printf("GPGPU-Sim PTX: in __cudaRegisterTexture:\n"); + gpu->gpgpu_ptx_sim_bindNameToTexture(devStr.data(), hostVar, dim, norm, ext); + printf("GPGPU-Sim PTX: int dim = %d\n", dim); + printf("GPGPU-Sim PTX: int norm = %d\n", norm); + printf("GPGPU-Sim PTX: int ext = %d\n", ext); + printf("GPGPU-Sim PTX: Execution warning: Not finished implementing \"%s\"\n", __my_func__ ); +} + +#ifndef OPENGL_SUPPORT +typedef unsigned long GLuint; +#endif + +cudaError_t cudaGLRegisterBufferObject(GLuint bufferObj) +{ + printf("GPGPU-Sim PTX: Execution warning: ignoring call to \"%s\"\n", __my_func__ ); + return g_last_cudaError = cudaSuccess; +} + +struct glbmap_entry { + GLuint m_bufferObj; + void *m_devPtr; + size_t m_size; + struct glbmap_entry *m_next; +}; +typedef struct glbmap_entry glbmap_entry_t; + +glbmap_entry_t* g_glbmap = NULL; + +cudaError_t cudaGLMapBufferObject(void** devPtr, GLuint bufferObj) +{ +#ifdef OPENGL_SUPPORT + GLint buffer_size=0; + CUctx_st* ctx = GPGPUSim_Context(); + + glbmap_entry_t *p = g_glbmap; + while ( p && p->m_bufferObj != bufferObj ) + p = p->m_next; + if ( p == NULL ) { + glBindBuffer(GL_ARRAY_BUFFER,bufferObj); + glGetBufferParameteriv(GL_ARRAY_BUFFER,GL_BUFFER_SIZE,&buffer_size); + assert( buffer_size != 0 ); + *devPtr = ctx->get_device()->get_gpgpu()->gpu_malloc(buffer_size); + + // create entry and insert to front of list + glbmap_entry_t *n = (glbmap_entry_t *) calloc(1,sizeof(glbmap_entry_t)); + n->m_next = g_glbmap; + g_glbmap = n; + + // initialize entry + n->m_bufferObj = bufferObj; + n->m_devPtr = *devPtr; + n->m_size = buffer_size; + + p = n; + } else { + buffer_size = p->m_size; + *devPtr = p->m_devPtr; + } + + if ( *devPtr ) { + char *data = (char *) calloc(p->m_size,1); + glGetBufferSubData(GL_ARRAY_BUFFER,0,buffer_size,data); + memcpy_to_gpu( (size_t) *devPtr, data, buffer_size ); + free(data); + printf("GPGPU-Sim PTX: cudaGLMapBufferObject %zu bytes starting at 0x%llx..\n", (size_t)buffer_size, + (unsigned long long) *devPtr); + return g_last_cudaError = cudaSuccess; + } else { + return g_last_cudaError = cudaErrorMemoryAllocation; + } + + return g_last_cudaError = cudaSuccess; +#else + fflush(stdout); + fflush(stderr); + printf("GPGPU-Sim PTX: GPGPU-Sim support for OpenGL integration disabled -- exiting\n"); + fflush(stdout); + exit(50); +#endif +} + +cudaError_t cudaGLUnmapBufferObject(GLuint bufferObj) +{ +#ifdef OPENGL_SUPPORT + glbmap_entry_t *p = g_glbmap; + while ( p && p->m_bufferObj != bufferObj ) + p = p->m_next; + if ( p == NULL ) + return g_last_cudaError = cudaErrorUnknown; + + char *data = (char *) calloc(p->m_size,1); + memcpy_from_gpu( data,(size_t)p->m_devPtr,p->m_size ); + glBufferSubData(GL_ARRAY_BUFFER,0,p->m_size,data); + free(data); + + return g_last_cudaError = cudaSuccess; +#else + fflush(stdout); + fflush(stderr); + printf("GPGPU-Sim PTX: support for OpenGL integration disabled -- exiting\n"); + fflush(stdout); + exit(50); +#endif +} + +cudaError_t cudaGLUnregisterBufferObject(GLuint bufferObj) +{ + printf("GPGPU-Sim PTX: Execution warning: ignoring call to \"%s\"\n", __my_func__ ); + return g_last_cudaError = cudaSuccess; +} + +#if (CUDART_VERSION >= 2010) + +cudaError_t CUDARTAPI cudaHostAlloc(void **pHost, size_t bytes, unsigned int flags) +{ + *pHost = malloc(bytes); + //need to track the size allocated so that cudaHostGetDevicePointer() can function properly. + //TODO: vary this function behavior based on flags value (following nvidia documentation) + pinned_memory_size[*pHost]=bytes; + if( *pHost ) + return g_last_cudaError = cudaSuccess; + else + return g_last_cudaError = cudaErrorMemoryAllocation; +} + +cudaError_t CUDARTAPI cudaHostGetDevicePointer(void **pDevice, void *pHost, unsigned int flags) +{ + //only cpu memory allocation happens in cudaHostAlloc. Linking with device pointer to pinned memory happens here. + //TODO: once kernel is executed, the contents in global pointer of GPU must be copied back to CPU host pointer! + flags=0; + CUctx_st* context = GPGPUSim_Context(); + gpgpu_t *gpu = context->get_device()->get_gpgpu(); + std::map::const_iterator i = pinned_memory_size.find(pHost); + assert(i != pinned_memory_size.end()); + size_t size = i->second; + *pDevice = gpu->gpu_malloc(size); + if(g_debug_execution >= 3) + printf("GPGPU-Sim PTX: cudaMallocing %zu bytes starting at 0x%llx..\n",size, (unsigned long long) *pDevice); + if ( *pDevice ) { + pinned_memory[pHost]=pDevice; + //Copy contents in cpu to gpu + gpu->memcpy_to_gpu((size_t)*pDevice,pHost,size); + return g_last_cudaError = cudaSuccess; + } else { + return g_last_cudaError = cudaErrorMemoryAllocation; + } +} + +cudaError_t CUDARTAPI cudaSetValidDevices(int *device_arr, int len) +{ + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; +} + +cudaError_t CUDARTAPI cudaSetDeviceFlags( int flags ) +{ + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; +} + +cudaError_t CUDARTAPI cudaFuncGetAttributes(struct cudaFuncAttributes *attr, const char *hostFun ) +{ + CUctx_st *context = GPGPUSim_Context(); + function_info *entry = context->get_kernel(hostFun); + if( entry ) { + const struct gpgpu_ptx_sim_info *kinfo = entry->get_kernel_info(); + attr->sharedSizeBytes = kinfo->smem; + attr->constSizeBytes = kinfo->cmem; + attr->localSizeBytes = kinfo->lmem; + attr->numRegs = kinfo->regs; + attr->maxThreadsPerBlock = 0; // from pragmas? +#if CUDART_VERSION >= 3000 + attr->ptxVersion = kinfo->ptx_version; + attr->binaryVersion = kinfo->sm_target; +#endif + } + return g_last_cudaError = cudaSuccess; +} + +cudaError_t CUDARTAPI cudaEventCreateWithFlags(cudaEvent_t *event, int flags) +{ + CUevent_st *e = new CUevent_st(flags==cudaEventBlockingSync); + g_timer_events[e->get_uid()] = e; +#if CUDART_VERSION >= 3000 + *event = e; +#else + *event = e->get_uid(); +#endif + return g_last_cudaError = cudaSuccess; +} + +cudaError_t CUDARTAPI cudaDriverGetVersion(int *driverVersion) +{ + *driverVersion = CUDART_VERSION; + return g_last_cudaError = cudaErrorUnknown; +} + +cudaError_t CUDARTAPI cudaRuntimeGetVersion(int *runtimeVersion) +{ + *runtimeVersion = CUDART_VERSION; + return g_last_cudaError = cudaErrorUnknown; +} + +#if CUDART_VERSION >= 3000 +__host__ cudaError_t CUDARTAPI cudaFuncSetCacheConfig(const char *func, enum cudaFuncCache cacheConfig ) +{ + CUctx_st *context = GPGPUSim_Context(); + context->get_device()->get_gpgpu()->set_cache_config(context->get_kernel(func)->get_name(), (FuncCache)cacheConfig); + return g_last_cudaError = cudaSuccess; +} + +//Jin: hack for cdp +__host__ cudaError_t CUDARTAPI cudaDeviceSetLimit(enum cudaLimit limit, size_t value) { + return g_last_cudaError = cudaSuccess; +} +#endif + +#endif + +cudaError_t CUDARTAPI cudaGLSetGLDevice(int device) +{ + printf("GPGPU-Sim PTX: Execution warning: ignoring call to \"%s\"\n", __my_func__ ); + return g_last_cudaError = cudaErrorUnknown; +} + +typedef void* HGPUNV; + +cudaError_t CUDARTAPI cudaWGLGetDevice(int *device, HGPUNV hGpu) +{ + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; +} + +void CUDARTAPI __cudaMutexOperation(int lock) +{ + cuda_not_implemented(__my_func__,__LINE__); +} + +void CUDARTAPI __cudaTextureFetch(const void *tex, void *index, int integer, void *val) +{ + cuda_not_implemented(__my_func__,__LINE__); +} + +} + +namespace cuda_math { + +void CUDARTAPI __cudaMutexOperation(int lock) +{ + cuda_not_implemented(__my_func__,__LINE__); +} + +void CUDARTAPI __cudaTextureFetch(const void *tex, void *index, int integer, void *val) +{ + cuda_not_implemented(__my_func__,__LINE__); +} + +int CUDARTAPI __cudaSynchronizeThreads(void**, void*) +{ + //TODO This function should syncronize if we support Asyn kernel calls + return g_last_cudaError = cudaSuccess; +} + +} + +//////// + +extern int ptx_parse(); +extern int ptx__scan_string(const char*); +extern FILE *ptx_in; + +extern int ptxinfo_parse(); +extern int ptxinfo_debug; +extern FILE *ptxinfo_in; + +/// static functions + +static int load_static_globals( symbol_table *symtab, unsigned min_gaddr, unsigned max_gaddr, gpgpu_t *gpu ) +{ + printf( "GPGPU-Sim PTX: loading globals with explicit initializers... \n" ); + fflush(stdout); + int ng_bytes=0; + symbol_table::iterator g=symtab->global_iterator_begin(); + + for ( ; g!=symtab->global_iterator_end(); g++) { + symbol *global = *g; + if ( global->has_initializer() ) { + printf( "GPGPU-Sim PTX: initializing '%s' ... ", global->name().c_str() ); + unsigned addr=global->get_address(); + const type_info *type = global->type(); + type_info_key ti=type->get_key(); + size_t size; + int t; + ti.type_decode(size,t); + int nbytes = size/8; + int offset=0; + std::list init_list = global->get_initializer(); + for ( std::list::iterator i=init_list.begin(); i!=init_list.end(); i++ ) { + operand_info op = *i; + ptx_reg_t value = op.get_literal_value(); + assert( (addr+offset+nbytes) < min_gaddr ); // min_gaddr is start of "heap" for cudaMalloc + gpu->get_global_memory()->write(addr+offset,nbytes,&value,NULL,NULL); // assuming little endian here + offset+=nbytes; + ng_bytes+=nbytes; + } + printf(" wrote %u bytes\n", offset ); + } + } + printf( "GPGPU-Sim PTX: finished loading globals (%u bytes total).\n", ng_bytes ); + fflush(stdout); + return ng_bytes; +} + +static int load_constants( symbol_table *symtab, addr_t min_gaddr, gpgpu_t *gpu ) +{ + printf( "GPGPU-Sim PTX: loading constants with explicit initializers... " ); + fflush(stdout); + int nc_bytes = 0; + symbol_table::iterator g=symtab->const_iterator_begin(); + + for ( ; g!=symtab->const_iterator_end(); g++) { + symbol *constant = *g; + if ( constant->is_const() && constant->has_initializer() ) { + + // get the constant element data size + int basic_type; + size_t num_bits; + constant->type()->get_key().type_decode(num_bits,basic_type); + + std::list init_list = constant->get_initializer(); + int nbytes_written = 0; + for ( std::list::iterator i=init_list.begin(); i!=init_list.end(); i++ ) { + operand_info op = *i; + ptx_reg_t value = op.get_literal_value(); + int nbytes = num_bits/8; + switch ( op.get_type() ) { + case int_t: assert(nbytes >= 1); break; + case float_op_t: assert(nbytes == 4); break; + case double_op_t: assert(nbytes >= 4); break; // account for double DEMOTING + default: + abort(); + } + unsigned addr=constant->get_address() + nbytes_written; + assert( addr+nbytes < min_gaddr ); + + gpu->get_global_memory()->write(addr,nbytes,&value,NULL,NULL); // assume little endian (so u8 is the first byte in u32) + nc_bytes+=nbytes; + nbytes_written += nbytes; + } + } + } + printf( " done.\n"); + fflush(stdout); + return nc_bytes; +} + +kernel_info_t *gpgpu_cuda_ptx_sim_init_grid( const char *hostFun, + gpgpu_ptx_sim_arg_list_t args, + struct dim3 gridDim, + struct dim3 blockDim, + CUctx_st* context ) +{ + function_info *entry = context->get_kernel(hostFun); + kernel_info_t *result = new kernel_info_t(gridDim,blockDim,entry); + if( entry == NULL ) { + printf("GPGPU-Sim PTX: ERROR launching kernel -- no PTX implementation found for %p\n", hostFun); + abort(); + } + unsigned argcount=args.size(); + unsigned argn=1; + for( gpgpu_ptx_sim_arg_list_t::iterator a = args.begin(); a != args.end(); a++ ) { + entry->add_param_data(argcount-argn,&(*a)); + argn++; + } + + entry->finalize(result->get_param_memory()); + g_ptx_kernel_count++; + fflush(stdout); + + return result; +} diff --git a/src/cuda-sim/cuda-sim.cc~ b/src/cuda-sim/cuda-sim.cc~ new file mode 100644 index 0000000..cfd901f --- /dev/null +++ b/src/cuda-sim/cuda-sim.cc~ @@ -0,0 +1,2155 @@ +// Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda, Wilson W.L. Fung, +// George L. Yuan, Jimmy Kwa +// The University of British Columbia +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// Redistributions in binary form must reproduce the above copyright notice, this +// list of conditions and the following disclaimer in the documentation and/or +// other materials provided with the distribution. +// Neither the name of The University of British Columbia nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "cuda-sim.h" + +#include "instructions.h" +#include "ptx_ir.h" +#include "ptx.tab.h" +#include "ptx_sim.h" +#include + +#include "opcodes.h" +#include "../statwrapper.h" +#include +#include +#include "../abstract_hardware_model.h" +#include "memory.h" +#include "ptx-stats.h" +#include "ptx_loader.h" +#include "ptx_parser.h" +#include "../gpgpu-sim/gpu-sim.h" +#include "ptx_sim.h" +#include "../gpgpusim_entrypoint.h" +#include "decuda_pred_table/decuda_pred_table.h" +#include "../stream_manager.h" +#include "cuda_device_runtime.h" + +int gpgpu_ptx_instruction_classification; +void ** g_inst_classification_stat = NULL; +void ** g_inst_op_classification_stat= NULL; +int g_ptx_kernel_count = -1; // used for classification stat collection purposes +int g_debug_execution = 0; +int g_debug_thread_uid = 0; +addr_t g_debug_pc = 0xBEEF1518; +// Output debug information to file options + +unsigned g_ptx_sim_num_insn = 0; +unsigned gpgpu_param_num_shaders = 0; + +char *opcode_latency_int, *opcode_latency_fp, *opcode_latency_dp; +char *opcode_initiation_int, *opcode_initiation_fp, *opcode_initiation_dp; +char *cdp_latency_str; +unsigned cdp_latency[5]; + +void ptx_opcocde_latency_options (option_parser_t opp) { + option_parser_register(opp, "-ptx_opcode_latency_int", OPT_CSTR, &opcode_latency_int, + "Opcode latencies for integers " + "Default 1,1,19,25,145", + "1,1,19,25,145"); + option_parser_register(opp, "-ptx_opcode_latency_fp", OPT_CSTR, &opcode_latency_fp, + "Opcode latencies for single precision floating points " + "Default 1,1,1,1,30", + "1,1,1,1,30"); + option_parser_register(opp, "-ptx_opcode_latency_dp", OPT_CSTR, &opcode_latency_dp, + "Opcode latencies for double precision floating points " + "Default 8,8,8,8,335", + "8,8,8,8,335"); + option_parser_register(opp, "-ptx_opcode_initiation_int", OPT_CSTR, &opcode_initiation_int, + "Opcode initiation intervals for integers " + "Default 1,1,4,4,32", + "1,1,4,4,32"); + option_parser_register(opp, "-ptx_opcode_initiation_fp", OPT_CSTR, &opcode_initiation_fp, + "Opcode initiation intervals for single precision floating points " + "Default 1,1,1,1,5", + "1,1,1,1,5"); + option_parser_register(opp, "-ptx_opcode_initiation_dp", OPT_CSTR, &opcode_initiation_dp, + "Opcode initiation intervals for double precision floating points " + "Default 8,8,8,8,130", + "8,8,8,8,130"); + option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str, + "CDP API latency " + "Default 7200,8000,100,12000,1600", + "7200,8000,100,12000,1600"); +} + +static address_type get_converge_point(address_type pc); + +void gpgpu_t::gpgpu_ptx_sim_bindNameToTexture(const char* name, const struct textureReference* texref, int dim, int readmode, int ext) +{ + std::string texname(name); + m_NameToTextureRef[texname] = texref; + const textureReferenceAttr *texAttr = new textureReferenceAttr(texref, dim, (enum cudaTextureReadMode)readmode, ext); + m_TextureRefToAttribute[texref] = texAttr; +} + +const char* gpgpu_t::gpgpu_ptx_sim_findNamefromTexture(const struct textureReference* texref) +{ + std::map::iterator itr = m_NameToTextureRef.begin(); + while (itr != m_NameToTextureRef.end()) { + if ((*itr).second == texref) { + const char *p = ((*itr).first).c_str(); + return p; + } + itr++; + } + return NULL; +} + +unsigned int intLOGB2( unsigned int v ) { + unsigned int shift; + unsigned int r; + + r = 0; + + shift = (( v & 0xFFFF0000) != 0 ) << 4; v >>= shift; r |= shift; + shift = (( v & 0xFF00 ) != 0 ) << 3; v >>= shift; r |= shift; + shift = (( v & 0xF0 ) != 0 ) << 2; v >>= shift; r |= shift; + shift = (( v & 0xC ) != 0 ) << 1; v >>= shift; r |= shift; + shift = (( v & 0x2 ) != 0 ) << 0; v >>= shift; r |= shift; + + return r; +} + +void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* texref, const struct cudaArray* array) +{ + m_TextureRefToCudaArray[texref] = array; + unsigned int texel_size_bits = array->desc.w + array->desc.x + array->desc.y + array->desc.z; + unsigned int texel_size = texel_size_bits/8; + unsigned int Tx, Ty; + int r; + + printf("GPGPU-Sim PTX: texel size = %d\n", texel_size); + printf("GPGPU-Sim PTX: texture cache linesize = %d\n", m_function_model_config.get_texcache_linesize()); + //first determine base Tx size for given linesize + switch (m_function_model_config.get_texcache_linesize()) { + case 16: Tx = 4; break; + case 32: Tx = 8; break; + case 64: Tx = 8; break; + case 128: Tx = 16; break; + case 256: Tx = 16; break; + default: + printf("GPGPU-Sim PTX: Line size of %d bytes currently not supported.\n", m_function_model_config.get_texcache_linesize()); + assert(0); + break; + } + r = texel_size >> 2; + //modify base Tx size to take into account size of each texel in bytes + while (r != 0) { + Tx = Tx >> 1; + r = r >> 2; + } + //by now, got the correct Tx size, calculate correct Ty size + Ty = m_function_model_config.get_texcache_linesize()/(Tx*texel_size); + + printf("GPGPU-Sim PTX: Tx = %d; Ty = %d, Tx_numbits = %d, Ty_numbits = %d\n", Tx, Ty, intLOGB2(Tx), intLOGB2(Ty)); + printf("GPGPU-Sim PTX: Texel size = %d bytes; texel_size_numbits = %d\n", texel_size, intLOGB2(texel_size)); + printf("GPGPU-Sim PTX: Binding texture to array starting at devPtr32 = 0x%x\n", array->devPtr32); + printf("GPGPU-Sim PTX: Texel size = %d bytes\n", texel_size); + struct textureInfo* texInfo = (struct textureInfo*) malloc(sizeof(struct textureInfo)); + texInfo->Tx = Tx; + texInfo->Ty = Ty; + texInfo->Tx_numbits = intLOGB2(Tx); + texInfo->Ty_numbits = intLOGB2(Ty); + texInfo->texel_size = texel_size; + texInfo->texel_size_numbits = intLOGB2(texel_size); + m_TextureRefToTexureInfo[texref] = texInfo; +} + +unsigned g_assemble_code_next_pc=0; +std::map g_pc_to_finfo; +std::vector function_info::s_g_pc_to_insn; + +#define MAX_INST_SIZE 8 /*bytes*/ + +void function_info::ptx_assemble() +{ + if( m_assembled ) { + return; + } + + // get the instructions into instruction memory... + unsigned num_inst = m_instructions.size(); + m_instr_mem_size = MAX_INST_SIZE*(num_inst+1); + m_instr_mem = new ptx_instruction*[ m_instr_mem_size ]; + + printf("GPGPU-Sim PTX: instruction assembly for function \'%s\'... ", m_name.c_str() ); + fflush(stdout); + std::list::iterator i; + + addr_t PC = g_assemble_code_next_pc; // globally unique address (across functions) + // start function on an aligned address + for( unsigned i=0; i < (PC%MAX_INST_SIZE); i++ ) + s_g_pc_to_insn.push_back((ptx_instruction*)NULL); + PC += PC%MAX_INST_SIZE; + m_start_PC = PC; + + addr_t n=0; // offset in m_instr_mem + //Why s_g_pc_to_insn.size() is needed to reserve additional memory for insts? reserve is cumulative. + //s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + MAX_INST_SIZE*m_instructions.size()); + s_g_pc_to_insn.reserve(MAX_INST_SIZE*m_instructions.size()); + for ( i=m_instructions.begin(); i != m_instructions.end(); i++ ) { + ptx_instruction *pI = *i; + if ( pI->is_label() ) { + const symbol *l = pI->get_label(); + labels[l->name()] = n; + } else { + g_pc_to_finfo[PC] = this; + m_instr_mem[n] = pI; + s_g_pc_to_insn.push_back(pI); + assert(pI == s_g_pc_to_insn[PC]); + pI->set_m_instr_mem_index(n); + pI->set_PC(PC); + assert( pI->inst_size() <= MAX_INST_SIZE ); + for( unsigned i=1; i < pI->inst_size(); i++ ) { + s_g_pc_to_insn.push_back((ptx_instruction*)NULL); + m_instr_mem[n+i]=NULL; + } + n += pI->inst_size(); + PC += pI->inst_size(); + } + } + g_assemble_code_next_pc=PC; + for ( unsigned ii=0; ii < n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions + ptx_instruction *pI = m_instr_mem[ii]; + if ( pI->get_opcode() == BRA_OP || pI->get_opcode() == BREAKADDR_OP || pI->get_opcode() == CALLP_OP) { + operand_info &target = pI->dst(); //get operand, e.g. target name + if ( labels.find(target.name()) == labels.end() ) { + printf("GPGPU-Sim PTX: Loader error (%s:%u): Branch label \"%s\" does not appear in assembly code.", + pI->source_file(),pI->source_line(), target.name().c_str() ); + abort(); + } + unsigned index = labels[ target.name() ]; //determine address from name + unsigned PC = m_instr_mem[index]->get_PC(); + m_symtab->set_label_address( target.get_symbol(), PC ); + target.set_type(label_t); + } + } + m_n = n; + printf(" done.\n"); + fflush(stdout); + + //disable pdom analysis here and do it at runtime + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", m_name.c_str() ); + create_basic_blocks(); + connect_basic_blocks(); + bool modified = false; + do { + find_dominators(); + find_idominators(); + modified = connect_break_targets(); + } while (modified == true); + + if ( g_debug_execution>=50 ) { + print_basic_blocks(); + print_basic_block_links(); + print_basic_block_dot(); + } + if ( g_debug_execution>=2 ) { + print_dominators(); + } + find_postdominators(); + find_ipostdominators(); + if ( g_debug_execution>=50 ) { + print_postdominators(); + print_ipostdominators(); + } + + printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", m_name.c_str() ); + for ( unsigned ii=0; ii < n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions + ptx_instruction *pI = m_instr_mem[ii]; + pI->pre_decode(); + } + printf("GPGPU-Sim PTX: ... done pre-decoding instructions for \'%s\'.\n", m_name.c_str() ); + fflush(stdout); + + m_assembled = true; +} + +addr_t shared_to_generic( unsigned smid, addr_t addr ) +{ + assert( addr < SHARED_MEM_SIZE_MAX ); + return SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX + addr; +} + +addr_t global_to_generic( addr_t addr ) +{ + return addr; +} + +bool isspace_shared( unsigned smid, addr_t addr ) +{ + addr_t start = SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX; + addr_t end = SHARED_GENERIC_START + (smid+1)*SHARED_MEM_SIZE_MAX; + if( (addr >= end) || (addr < start) ) + return false; + return true; +} + +bool isspace_global( addr_t addr ) +{ + return (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT); +} + +memory_space_t whichspace( addr_t addr ) +{ + if( (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT) ) { + return global_space; + } else if( addr >= SHARED_GENERIC_START ) { + return shared_space; + } else { + return local_space; + } +} + +addr_t generic_to_shared( unsigned smid, addr_t addr ) +{ + assert(isspace_shared(smid,addr)); + return addr - (SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX); +} + +addr_t local_to_generic( unsigned smid, unsigned hwtid, addr_t addr ) +{ + assert(addr < LOCAL_MEM_SIZE_MAX); + return LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid) + addr; +} + +bool isspace_local( unsigned smid, unsigned hwtid, addr_t addr ) +{ + addr_t start = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid); + addr_t end = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * (hwtid+1)); + if( (addr >= end) || (addr < start) ) + return false; + return true; +} + +addr_t generic_to_local( unsigned smid, unsigned hwtid, addr_t addr ) +{ + assert(isspace_local(smid,hwtid,addr)); + return addr - (LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid)); +} + +addr_t generic_to_global( addr_t addr ) +{ + return addr; +} + + +void* gpgpu_t::gpu_malloc( size_t size ) +{ + unsigned long long result = m_dev_malloc; + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address 0x%Lx\n", size, m_dev_malloc ); + fflush(stdout); + } + m_dev_malloc += size; + if (size%256) m_dev_malloc += (256 - size%256); //align to 256 byte boundaries + return(void*) result; +} + +void* gpgpu_t::gpu_mallocarray( size_t size ) +{ + unsigned long long result = m_dev_malloc; + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address 0x%Lx\n", size, m_dev_malloc ); + fflush(stdout); + } + m_dev_malloc += size; + if (size%256) m_dev_malloc += (256 - size%256); //align to 256 byte boundaries + return(void*) result; +} + + +void gpgpu_t::memcpy_to_gpu( size_t dst_start_addr, const void *src, size_t count ) +{ + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: copying %zu bytes from CPU[0x%Lx] to GPU[0x%Lx] ... ", count, (unsigned long long) src, (unsigned long long) dst_start_addr ); + fflush(stdout); + } + char *src_data = (char*)src; + for (unsigned n=0; n < count; n ++ ) + m_global_mem->write(dst_start_addr+n,1, src_data+n,NULL,NULL); + if(g_debug_execution >= 3) { + printf( " done.\n"); + fflush(stdout); + } +} + +void gpgpu_t::memcpy_from_gpu( void *dst, size_t src_start_addr, size_t count ) +{ + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to CPU[0x%Lx] ...", count, (unsigned long long) src_start_addr, (unsigned long long) dst ); + fflush(stdout); + } + unsigned char *dst_data = (unsigned char*)dst; + for (unsigned n=0; n < count; n ++ ) + m_global_mem->read(src_start_addr+n,1,dst_data+n); + if(g_debug_execution >= 3) { + printf( " done.\n"); + fflush(stdout); + } +} + +void gpgpu_t::memcpy_gpu_to_gpu( size_t dst, size_t src, size_t count ) +{ + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to GPU[0x%Lx] ...", count, + (unsigned long long) src, (unsigned long long) dst ); + fflush(stdout); + } + for (unsigned n=0; n < count; n ++ ) { + unsigned char tmp; + m_global_mem->read(src+n,1,&tmp); + m_global_mem->write(dst+n,1, &tmp,NULL,NULL); + } + if(g_debug_execution >= 3) { + printf( " done.\n"); + fflush(stdout); + } +} + +void gpgpu_t::gpu_memset( size_t dst_start_addr, int c, size_t count ) +{ + if(g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: setting %zu bytes of memory to 0x%x starting at 0x%Lx... ", + count, (unsigned char) c, (unsigned long long) dst_start_addr ); + fflush(stdout); + } + unsigned char c_value = (unsigned char)c; + for (unsigned n=0; n < count; n ++ ) + m_global_mem->write(dst_start_addr+n,1,&c_value,NULL,NULL); + if(g_debug_execution >= 3) { + printf( " done.\n"); + fflush(stdout); + } +} + +void ptx_print_insn( address_type pc, FILE *fp ) +{ + std::map::iterator f = g_pc_to_finfo.find(pc); + if( f == g_pc_to_finfo.end() ) { + fprintf(fp,"", pc ); + return; + } + function_info *finfo = f->second; + assert( finfo ); + finfo->print_insn(pc,fp); +} + +std::string ptx_get_insn_str( address_type pc ) +{ + std::map::iterator f = g_pc_to_finfo.find(pc); + if( f == g_pc_to_finfo.end() ) { + #define STR_SIZE 255 + char buff[STR_SIZE]; + buff[STR_SIZE - 1] = '\0'; + snprintf(buff, STR_SIZE,"", pc ); + return std::string(buff); + } + function_info *finfo = f->second; + assert( finfo ); + return finfo->get_insn_str(pc); +} + +void ptx_instruction::set_fp_or_int_archop(){ + oprnd_type=UN_OP; + if((m_opcode == MEMBAR_OP)||(m_opcode == SSY_OP )||(m_opcode == BRA_OP) || (m_opcode == BAR_OP) || (m_opcode == RET_OP) || (m_opcode == RETP_OP) || (m_opcode == NOP_OP) || (m_opcode == EXIT_OP) || (m_opcode == CALLP_OP) || (m_opcode == CALL_OP)){ + // do nothing + }else if((m_opcode == CVT_OP || m_opcode == SET_OP || m_opcode == SLCT_OP)){ + if(get_type2()==F16_TYPE || get_type2()==F32_TYPE || get_type2() == F64_TYPE || get_type2() == FF64_TYPE){ + oprnd_type= FP_OP; + }else oprnd_type=INT_OP; + + }else{ + if(get_type()==F16_TYPE || get_type()==F32_TYPE || get_type() == F64_TYPE || get_type() == FF64_TYPE){ + oprnd_type= FP_OP; + }else oprnd_type=INT_OP; + } +} +void ptx_instruction::set_mul_div_or_other_archop(){ + sp_op=OTHER_OP; + if((m_opcode != MEMBAR_OP) && (m_opcode != SSY_OP) && (m_opcode != BRA_OP) && (m_opcode != BAR_OP) && (m_opcode != EXIT_OP) && (m_opcode != NOP_OP) && (m_opcode != RETP_OP) && (m_opcode != RET_OP) && (m_opcode != CALLP_OP) && (m_opcode != CALL_OP)){ + if(get_type()==F32_TYPE || get_type() == F64_TYPE || get_type() == FF64_TYPE){ + switch(get_opcode()){ + case MUL_OP: + case MAD_OP: + sp_op=FP_MUL_OP; + break; + case DIV_OP: + sp_op=FP_DIV_OP; + break; + case LG2_OP: + sp_op=FP_LG_OP; + break; + case RSQRT_OP: + case SQRT_OP: + sp_op=FP_SQRT_OP; + break; + case RCP_OP: + sp_op=FP_DIV_OP; + break; + case SIN_OP: + case COS_OP: + sp_op=FP_SIN_OP; + break; + case EX2_OP: + sp_op=FP_EXP_OP; + break; + default: + if(op==ALU_OP) + sp_op=FP__OP; + break; + + } + }else { + switch(get_opcode()){ + case MUL24_OP: + case MAD24_OP: + sp_op=INT_MUL24_OP; + break; + case MUL_OP: + case MAD_OP: + if(get_type()==U32_TYPE || get_type()==S32_TYPE || get_type()==B32_TYPE) + sp_op=INT_MUL32_OP; + else + sp_op=INT_MUL_OP; + break; + case DIV_OP: + sp_op=INT_DIV_OP; + break; + default: + if(op==ALU_OP) + sp_op=INT__OP; + break; + } + } + } + +} + + + +void ptx_instruction::set_bar_type() +{ + if(m_opcode==BAR_OP) { + switch(m_barrier_op){ + case SYNC_OPTION: + bar_type = SYNC; + break; + case ARRIVE_OPTION: + bar_type = ARRIVE; + break; + case RED_OPTION: + bar_type = RED; + switch(m_atomic_spec){ + case ATOMIC_POPC: + red_type = POPC_RED; + break; + case ATOMIC_AND: + red_type = AND_RED; + break; + case ATOMIC_OR: + red_type = OR_RED; + break; + } + break; + default: + abort(); + } + } +} + + +void ptx_instruction::set_opcode_and_latency() +{ + unsigned int_latency[5]; + unsigned fp_latency[5]; + unsigned dp_latency[5]; + unsigned int_init[5]; + unsigned fp_init[5]; + unsigned dp_init[5]; + /* + * [0] ADD,SUB + * [1] MAX,Min + * [2] MUL + * [3] MAD + * [4] DIV + */ + sscanf(opcode_latency_int, "%u,%u,%u,%u,%u", + &int_latency[0],&int_latency[1],&int_latency[2], + &int_latency[3],&int_latency[4]); + sscanf(opcode_latency_fp, "%u,%u,%u,%u,%u", + &fp_latency[0],&fp_latency[1],&fp_latency[2], + &fp_latency[3],&fp_latency[4]); + sscanf(opcode_latency_dp, "%u,%u,%u,%u,%u", + &dp_latency[0],&dp_latency[1],&dp_latency[2], + &dp_latency[3],&dp_latency[4]); + sscanf(opcode_initiation_int, "%u,%u,%u,%u,%u", + &int_init[0],&int_init[1],&int_init[2], + &int_init[3],&int_init[4]); + sscanf(opcode_initiation_fp, "%u,%u,%u,%u,%u", + &fp_init[0],&fp_init[1],&fp_init[2], + &fp_init[3],&fp_init[4]); + sscanf(opcode_initiation_dp, "%u,%u,%u,%u,%u", + &dp_init[0],&dp_init[1],&dp_init[2], + &dp_init[3],&dp_init[4]); + sscanf(cdp_latency_str, "%u,%u,%u,%u,%u", + &cdp_latency[0],&cdp_latency[1],&cdp_latency[2], + &cdp_latency[3],&cdp_latency[4]); + + if(!m_operands.empty()){ + std::vector::iterator it; + for(it=++m_operands.begin();it!=m_operands.end();it++){ + num_operands++; + if((it->is_reg() || it->is_vector())){ + num_regs++; + } + } + } + op = ALU_OP; + mem_op= NOT_TEX; + initiation_interval = latency = 1; + switch( m_opcode ) { + case MOV_OP: + assert( !(has_memory_read() && has_memory_write()) ); + if ( has_memory_read() ) op = LOAD_OP; + if ( has_memory_write() ) op = STORE_OP; + break; + case LD_OP: op = LOAD_OP; break; + case LDU_OP: op = LOAD_OP; break; + case ST_OP: op = STORE_OP; break; + case BRA_OP: op = BRANCH_OP; break; + case BREAKADDR_OP: op = BRANCH_OP; break; + case TEX_OP: op = LOAD_OP; mem_op=TEX; break; + case ATOM_OP: op = LOAD_OP; break; + case BAR_OP: op = BARRIER_OP; break; + case MEMBAR_OP: op = MEMORY_BARRIER_OP; break; + case CALL_OP: + { + if(m_is_printf || m_is_cdp) { + op = ALU_OP; + } + else + op = CALL_OPS; + break; + } + case CALLP_OP: + { + if(m_is_printf || m_is_cdp) { + op = ALU_OP; + } + else + op = CALL_OPS; + break; + } + case RET_OP: case RETP_OP: op = RET_OPS;break; + case ADD_OP: case ADDP_OP: case ADDC_OP: case SUB_OP: case SUBC_OP: + //ADD,SUB latency + switch(get_type()){ + case F32_TYPE: + latency = fp_latency[0]; + initiation_interval = fp_init[0]; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[0]; + initiation_interval = dp_init[0]; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: //Use int settings for default + latency = int_latency[0]; + initiation_interval = int_init[0]; + break; + } + break; + case MAX_OP: case MIN_OP: + //MAX,MIN latency + switch(get_type()){ + case F32_TYPE: + latency = fp_latency[1]; + initiation_interval = fp_init[1]; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[1]; + initiation_interval = dp_init[1]; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: //Use int settings for default + latency = int_latency[1]; + initiation_interval = int_init[1]; + break; + } + break; + case MUL_OP: + //MUL latency + switch(get_type()){ + case F32_TYPE: + latency = fp_latency[2]; + initiation_interval = fp_init[2]; + op = ALU_SFU_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[2]; + initiation_interval = dp_init[2]; + op = ALU_SFU_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: //Use int settings for default + latency = int_latency[2]; + initiation_interval = int_init[2]; + op = SFU_OP; + break; + } + break; + case MAD_OP: case MADC_OP: case MADP_OP: + //MAD latency + switch(get_type()){ + case F32_TYPE: + latency = fp_latency[3]; + initiation_interval = fp_init[3]; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[3]; + initiation_interval = dp_init[3]; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: //Use int settings for default + latency = int_latency[3]; + initiation_interval = int_init[3]; + op = SFU_OP; + break; + } + break; + case DIV_OP: + // Floating point only + op = SFU_OP; + switch(get_type()){ + case F32_TYPE: + latency = fp_latency[4]; + initiation_interval = fp_init[4]; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[4]; + initiation_interval = dp_init[4]; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: //Use int settings for default + latency = int_latency[4]; + initiation_interval = int_init[4]; + break; + } + break; + case SQRT_OP: case SIN_OP: case COS_OP: case EX2_OP: case LG2_OP: case RSQRT_OP: case RCP_OP: + //Using double to approximate those + latency = dp_latency[2]; + initiation_interval = dp_init[2]; + op = SFU_OP; + break; + case SHFL_OP: + latency = 32; + initiation_interval = 15; + break; + default: + break; + } + set_fp_or_int_archop(); + set_mul_div_or_other_archop(); + +} + +void ptx_thread_info::ptx_fetch_inst( inst_t &inst ) const +{ + addr_t pc = get_pc(); + const ptx_instruction *pI = m_func_info->get_instruction(pc); + inst = (const inst_t&)*pI; + assert( inst.valid() ); +} + +static unsigned datatype2size( unsigned data_type ) +{ + unsigned data_size; + switch ( data_type ) { + case B8_TYPE: + case S8_TYPE: + case U8_TYPE: + data_size = 1; break; + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: + case F16_TYPE: + data_size = 2; break; + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: + case F32_TYPE: + data_size = 4; break; + case B64_TYPE: + case BB64_TYPE: + case S64_TYPE: + case U64_TYPE: + case F64_TYPE: + case FF64_TYPE: + data_size = 8; break; + case BB128_TYPE: + data_size = 16; break; + default: assert(0); break; + } + return data_size; +} + +void ptx_instruction::pre_decode() +{ + pc = m_PC; + isize = m_inst_size; + for( unsigned i=0; i<4; i++) { + out[i] = 0; + in[i] = 0; + } + is_vectorin = 0; + is_vectorout = 0; + std::fill_n(arch_reg.src, MAX_REG_OPERANDS, -1); + std::fill_n(arch_reg.dst, MAX_REG_OPERANDS, -1); + pred = 0; + ar1 = 0; + ar2 = 0; + space = m_space_spec; + memory_op = no_memory_op; + data_size = 0; + if ( has_memory_read() || has_memory_write() ) { + unsigned to_type = get_type(); + data_size = datatype2size(to_type); + memory_op = has_memory_read() ? memory_load : memory_store; + } + + bool has_dst = false ; + + switch ( get_opcode() ) { +#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break; +#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break; +#include "opcodes.def" +#undef OP_DEF +#undef OP_W_DEF + default: + printf( "Execution error: Invalid opcode (0x%x)\n", get_opcode() ); + break; + } + + switch( m_cache_option ) { + case CA_OPTION: cache_op = CACHE_ALL; break; + case CG_OPTION: cache_op = CACHE_GLOBAL; break; + case CS_OPTION: cache_op = CACHE_STREAMING; break; + case LU_OPTION: cache_op = CACHE_LAST_USE; break; + case CV_OPTION: cache_op = CACHE_VOLATILE; break; + case WB_OPTION: cache_op = CACHE_WRITE_BACK; break; + case WT_OPTION: cache_op = CACHE_WRITE_THROUGH; break; + default: + if( m_opcode == LD_OP || m_opcode == LDU_OP ) + cache_op = CACHE_ALL; + else if( m_opcode == ST_OP ) + cache_op = CACHE_WRITE_BACK; + else if( m_opcode == ATOM_OP ) + cache_op = CACHE_GLOBAL; + break; + } + + set_opcode_and_latency(); + set_bar_type(); + // Get register operands + int n=0,m=0; + ptx_instruction::const_iterator opr=op_iter_begin(); + for ( ; opr != op_iter_end(); opr++, n++ ) { //process operands + const operand_info &o = *opr; + if ( has_dst && n==0 ) { + // Do not set the null register "_" as an architectural register + if ( o.is_reg() && !o.is_non_arch_reg() ) { + out[0] = o.reg_num(); + arch_reg.dst[0] = o.arch_reg_num(); + } else if ( o.is_vector() ) { + is_vectorin = 1; + unsigned num_elem = o.get_vect_nelem(); + if( num_elem >= 1 ) out[0] = o.reg1_num(); + if( num_elem >= 2 ) out[1] = o.reg2_num(); + if( num_elem >= 3 ) out[2] = o.reg3_num(); + if( num_elem >= 4 ) out[3] = o.reg4_num(); + for (int i = 0; i < num_elem; i++) + arch_reg.dst[i] = o.arch_reg_num(i); + } + } else { + if ( o.is_reg() && !o.is_non_arch_reg() ) { + int reg_num = o.reg_num(); + arch_reg.src[m] = o.arch_reg_num(); + switch ( m ) { + case 0: in[0] = reg_num; break; + case 1: in[1] = reg_num; break; + case 2: in[2] = reg_num; break; + default: break; + } + m++; + } else if ( o.is_vector() ) { + //assert(m == 0); //only support 1 vector operand (for textures) right now + is_vectorout = 1; + unsigned num_elem = o.get_vect_nelem(); + if( num_elem >= 1 ) in[0] = o.reg1_num(); + if( num_elem >= 2 ) in[1] = o.reg2_num(); + if( num_elem >= 3 ) in[2] = o.reg3_num(); + if( num_elem >= 4 ) in[3] = o.reg4_num(); + for (int i = 0; i < num_elem; i++) + arch_reg.src[i] = o.arch_reg_num(i); + m+=4; + } + } + } + + // Get predicate + if(has_pred()) { + const operand_info &p = get_pred(); + pred = p.reg_num(); + } + + // Get address registers inside memory operands. + // Assuming only one memory operand per instruction, + // and maximum of two address registers for one memory operand. + if( has_memory_read() || has_memory_write() ) { + ptx_instruction::const_iterator op=op_iter_begin(); + for ( ; op != op_iter_end(); op++, n++ ) { //process operands + const operand_info &o = *op; + + if(o.is_memory_operand()) { + // We do not support the null register as a memory operand + assert( !o.is_non_arch_reg() ); + + // Check PTXPlus-type operand + // memory operand with addressing (ex. s[0x4] or g[$r1]) + if(o.is_memory_operand2()) { + + // memory operand with one address register (ex. g[$r1+0x4] or s[$r2+=0x4]) + if(o.get_double_operand_type() == 0 || o.get_double_operand_type() == 3){ + ar1 = o.reg_num(); + arch_reg.src[4] = o.arch_reg_num(); + // TODO: address register in $r2+=0x4 should be an output register as well + } + // memory operand with two address register (ex. s[$r1+$r1] or g[$r1+=$r2]) + else if(o.get_double_operand_type() == 1 || o.get_double_operand_type() == 2) { + ar1 = o.reg1_num(); + arch_reg.src[4] = o.arch_reg_num(); + ar2 = o.reg2_num(); + arch_reg.src[5] = o.arch_reg_num(); + // TODO: first address register in $r1+=$r2 should be an output register as well + } + } + else if(o.is_immediate_address()){ + + } + // Regular PTX operand + else if (o.get_symbol()->type()->get_key().is_reg()) { // Memory operand contains a register + ar1 = o.reg_num(); + arch_reg.src[4] = o.arch_reg_num(); + } + + } + } + } + + // get reconvergence pc + reconvergence_pc = get_converge_point(pc); + + m_decoded=true; +} + +void function_info::add_param_name_type_size( unsigned index, std::string name, int type, size_t size, bool ptr, memory_space_t space ) +{ + unsigned parsed_index; + char buffer[2048]; + snprintf(buffer,2048,"%s_param_%%u", m_name.c_str() ); + int ntokens = sscanf(name.c_str(),buffer,&parsed_index); + if( ntokens == 1 ) { + assert( m_ptx_kernel_param_info.find(parsed_index) == m_ptx_kernel_param_info.end() ); + m_ptx_kernel_param_info[parsed_index] = param_info(name, type, size, ptr, space); + } else { + assert( m_ptx_kernel_param_info.find(index) == m_ptx_kernel_param_info.end() ); + m_ptx_kernel_param_info[index] = param_info(name, type, size, ptr, space); + } +} + +void function_info::add_param_data( unsigned argn, struct gpgpu_ptx_sim_arg *args ) +{ + const void *data = args->m_start; + + bool scratchpad_memory_param = false; // Is this parameter in CUDA shared memory or OpenCL local memory + + std::map::iterator i=m_ptx_kernel_param_info.find(argn); + if( i != m_ptx_kernel_param_info.end() ) { + if (i->second.is_ptr_shared()) { + assert(args->m_start == NULL && "OpenCL parameter pointer to local memory must have NULL as value"); + scratchpad_memory_param = true; + } else { + param_t tmp; + tmp.pdata = args->m_start; + tmp.size = args->m_nbytes; + tmp.offset = args->m_offset; + tmp.type = 0; + i->second.add_data(tmp); + i->second.add_offset((unsigned) args->m_offset); + } + } else { + scratchpad_memory_param = true; + } + + if (scratchpad_memory_param) { + // This should only happen for OpenCL: + // + // The LLVM PTX compiler in NVIDIA's driver (version 190.29) + // does not generate an argument in the function declaration + // for __constant arguments. + // + // The associated constant memory space can be allocated in two + // ways. It can be explicitly initialized in the .ptx file where + // it is declared. Or, it can be allocated using the clCreateBuffer + // on the host. In this later case, the .ptx file will contain + // a global declaration of the parameter, but it will have an unknown + // array size. Thus, the symbol's address will not be set and we need + // to set it here before executing the PTX. + + char buffer[2048]; + snprintf(buffer,2048,"%s_param_%u",m_name.c_str(),argn); + + symbol *p = m_symtab->lookup(buffer); + if( p == NULL ) { + printf("GPGPU-Sim PTX: ERROR ** could not locate symbol for \'%s\' : cannot bind buffer\n", buffer); + abort(); + } + if( data ) + p->set_address((addr_t)*(size_t*)data); + else { + // clSetKernelArg was passed NULL pointer for data... + // this is used for dynamically sized shared memory on NVIDIA platforms + bool is_ptr_shared = false; + if( i != m_ptx_kernel_param_info.end() ) { + is_ptr_shared = i->second.is_ptr_shared(); + } + + if( !is_ptr_shared and !p->is_shared() ) { + printf("GPGPU-Sim PTX: ERROR ** clSetKernelArg passed NULL but arg not shared memory\n"); + abort(); + } + unsigned num_bits = 8*args->m_nbytes; + printf("GPGPU-Sim PTX: deferred allocation of shared region for \"%s\" from 0x%x to 0x%x (shared memory space)\n", + p->name().c_str(), + m_symtab->get_shared_next(), + m_symtab->get_shared_next() + num_bits/8 ); + fflush(stdout); + assert( (num_bits%8) == 0 ); + addr_t addr = m_symtab->get_shared_next(); + addr_t addr_pad = num_bits ? (((num_bits/8) - (addr % (num_bits/8))) % (num_bits/8)) : 0; + p->set_address( addr+addr_pad ); + m_symtab->alloc_shared( num_bits/8 + addr_pad ); + } + } +} + +unsigned function_info::get_args_aligned_size() { + + if(m_args_aligned_size >= 0) + return m_args_aligned_size; + + unsigned param_address = 0; + unsigned int total_size = 0; + for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + + size_t arg_size = p.get_size() / 8; // size of param in bytes + total_size = (total_size + arg_size - 1) / arg_size * arg_size; //aligned + p.add_offset(total_size); + param->set_address(param_address + total_size); + total_size += arg_size; + } + + m_args_aligned_size = (total_size + 3) / 4 * 4; //final size aligned to word + + return m_args_aligned_size; + +} + + +void function_info::finalize( memory_space *param_mem ) +{ + unsigned param_address = 0; + for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + param_info &p = i->second; + if (p.is_ptr_shared()) continue; // Pointer to local memory: Should we pass the allocated shared memory address to the param memory space? + std::string name = p.get_name(); + int type = p.get_type(); + param_t param_value = p.get_value(); + param_value.type = type; + symbol *param = m_symtab->lookup(name.c_str()); + unsigned xtype = param->type()->get_key().scalar_type(); + assert(xtype==(unsigned)type); + size_t size; + size = param_value.size; // size of param in bytes + // assert(param_value.offset == param_address); + if( size != p.get_size() / 8) { + printf("GPGPU-Sim PTX: WARNING actual kernel paramter size = %zu bytes vs. formal size = %zu (using smaller of two)\n", + size, p.get_size()/8); + size = (size<(p.get_size()/8))?size:(p.get_size()/8); + } + // copy the parameter over word-by-word so that parameter that crosses a memory page can be copied over + //Jin: copy parameter using aligned rules + const size_t word_size = 4; + param_address = (param_address + size - 1) / size * size; //aligned with size + for (size_t idx = 0; idx < size; idx += word_size) { + const char *pdata = reinterpret_cast(param_value.pdata) + idx; // cast to char * for ptr arithmetic + param_mem->write(param_address + idx, word_size, pdata,NULL,NULL); + } + unsigned offset = p.get_offset(); + assert(offset == param_address); + param->set_address(param_address); + param_address += size; + } +} + +void function_info::param_to_shared( memory_space *shared_mem, symbol_table *symtab ) +{ + // TODO: call this only for PTXPlus with GT200 models + extern gpgpu_sim* g_the_gpu; + if (not g_the_gpu->get_config().convert_to_ptxplus()) return; + + // copies parameters into simulated shared memory + for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + param_info &p = i->second; + if (p.is_ptr_shared()) continue; // Pointer to local memory: Should we pass the allocated shared memory address to the param memory space? + std::string name = p.get_name(); + int type = p.get_type(); + param_t value = p.get_value(); + value.type = type; + symbol *param = symtab->lookup(name.c_str()); + unsigned xtype = param->type()->get_key().scalar_type(); + assert(xtype==(unsigned)type); + + int tmp; + size_t size; + unsigned offset = p.get_offset(); + type_info_key::type_decode(xtype,size,tmp); + + // Write to shared memory - offset + 0x10 + shared_mem->write(offset+0x10,size/8,value.pdata,NULL,NULL); + } +} + + +void function_info::list_param( FILE *fout ) const +{ + for( std::map::const_iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + const param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); + fprintf(fout, "%s: %#08x\n", name.c_str(), param_addr); + } + fflush(fout); +} + +template +bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc) +{ + if (g_debug_execution >= activate_level) { + // check each type of debug dump constraint to filter out dumps + if ( (g_debug_thread_uid != 0) && (thd_uid != (unsigned)g_debug_thread_uid) ) { + return false; + } + if ( (g_debug_pc != 0xBEEF1518) && (pc != g_debug_pc) ) { + return false; + } + + return true; + } + + return false; +} + +void init_inst_classification_stat() +{ + static std::set init; + if( init.find(g_ptx_kernel_count) != init.end() ) + return; + init.insert(g_ptx_kernel_count); + + #define MAX_CLASS_KER 1024 + char kernelname[MAX_CLASS_KER] =""; + if (!g_inst_classification_stat) g_inst_classification_stat = (void**)calloc(MAX_CLASS_KER, sizeof(void*)); + snprintf(kernelname, MAX_CLASS_KER, "Kernel %d Classification\n",g_ptx_kernel_count ); + assert( g_ptx_kernel_count < MAX_CLASS_KER ) ; // a static limit on number of kernels increase it if it fails! + g_inst_classification_stat[g_ptx_kernel_count] = StatCreate(kernelname,1,20); + if (!g_inst_op_classification_stat) g_inst_op_classification_stat = (void**)calloc(MAX_CLASS_KER, sizeof(void*)); + snprintf(kernelname, MAX_CLASS_KER, "Kernel %d OP Classification\n",g_ptx_kernel_count ); + g_inst_op_classification_stat[g_ptx_kernel_count] = StatCreate(kernelname,1,100); +} + +static unsigned get_tex_datasize( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &src1 = pI->src1(); //the name of the texture + std::string texname = src1.name(); + + gpgpu_t *gpu = thread->get_gpu(); + const struct textureReference* texref = gpu->get_texref(texname); + const struct textureInfo* texInfo = gpu->get_texinfo(texref); + + unsigned data_size = texInfo->texel_size; + return data_size; +} + +void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id) +{ + + bool skip = false; + int op_classification = 0; + addr_t pc = next_instr(); + assert( pc == inst.pc ); // make sure timing model and functional model are in sync + const ptx_instruction *pI = m_func_info->get_instruction(pc); + set_npc( pc + pI->inst_size() ); + + + try { + + clearRPC(); + m_last_set_operand_value.u64 = 0; + + if(is_done()) + { + printf("attempted to execute instruction on a thread that is already done.\n"); + assert(0); + } + + if ( g_debug_execution >= 6 || m_gpu->get_config().get_ptx_inst_debug_to_file()) { + if ( (g_debug_thread_uid==0) || (get_uid() == (unsigned)g_debug_thread_uid) ) { + + clear_modifiedregs(); + enable_debug_trace(); + } + } + + + if( pI->has_pred() ) { + const operand_info &pred = pI->get_pred(); + ptx_reg_t pred_value = get_operand_value(pred, pred, PRED_TYPE, this, 0); + if(pI->get_pred_mod() == -1) { + skip = (pred_value.pred & 0x0001) ^ pI->get_pred_neg(); //ptxplus inverts the zero flag + } else { + skip = !pred_lookup(pI->get_pred_mod(), pred_value.pred & 0x000F); + } + } + + if( skip ) { + inst.set_not_active(lane_id); + } else { + const ptx_instruction *pI_saved = pI; + ptx_instruction *pJ = NULL; + if( pI->get_opcode() == VOTE_OP ) { + pJ = new ptx_instruction(*pI); + *((warp_inst_t*)pJ) = inst; // copy active mask information + pI = pJ; + } + switch ( pI->get_opcode() ) { +#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,this); op_classification = CLASSIFICATION; break; +#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,get_core(),inst); op_classification = CLASSIFICATION; break; +#include "opcodes.def" +#undef OP_DEF +#undef OP_W_DEF + default: printf( "Execution error: Invalid opcode (0x%x)\n", pI->get_opcode() ); break; + } + delete pJ; + pI = pI_saved; + + // Run exit instruction if exit option included + if(pI->is_exit()) + exit_impl(pI,this); + } + + + + const gpgpu_functional_sim_config &config = m_gpu->get_config(); + + // Output instruction information to file and stdout + if( config.get_ptx_inst_debug_to_file() != 0 && + (config.get_ptx_inst_debug_thread_uid() == 0 || config.get_ptx_inst_debug_thread_uid() == get_uid()) ) { + fprintf(m_gpu->get_ptx_inst_debug_file(), + "[thd=%u] : (%s:%u - %s)\n", + get_uid(), + pI->source_file(), pI->source_line(), pI->get_source() ); + //fprintf(ptx_inst_debug_file, "has memory read=%d, has memory write=%d\n", pI->has_memory_read(), pI->has_memory_write()); + fflush(m_gpu->get_ptx_inst_debug_file()); + } + + if ( ptx_debug_exec_dump_cond<5>(get_uid(), pc) ) { + dim3 ctaid = get_ctaid(); + dim3 tid = get_tid(); + printf("%u [thd=%u][i=%u] : ctaid=(%u,%u,%u) tid=(%u,%u,%u) icount=%u [pc=%u] (%s:%u - %s) [0x%llx]\n", + g_ptx_sim_num_insn, + get_uid(), + pI->uid(), ctaid.x,ctaid.y,ctaid.z,tid.x,tid.y,tid.z, + get_icount(), + pc, pI->source_file(), pI->source_line(), pI->get_source(), + m_last_set_operand_value.u64 ); + fflush(stdout); + } + + addr_t insn_memaddr = 0xFEEBDAED; + memory_space_t insn_space = undefined_space; + _memory_op_t insn_memory_op = no_memory_op; + unsigned insn_data_size = 0; + if ( (pI->has_memory_read() || pI->has_memory_write()) ) { + insn_memaddr = last_eaddr(); + insn_space = last_space(); + unsigned to_type = pI->get_type(); + insn_data_size = datatype2size(to_type); + insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; + } + + if ( pI->get_opcode() == BAR_OP && pI->barrier_op() == RED_OPTION) { + inst.add_callback( lane_id, last_callback().function, last_callback().instruction, this,false /*not atomic*/); + } + + if ( pI->get_opcode() == ATOM_OP ) { + insn_memaddr = last_eaddr(); + insn_space = last_space(); + inst.add_callback( lane_id, last_callback().function, last_callback().instruction, this,true /*atomic*/); + unsigned to_type = pI->get_type(); + insn_data_size = datatype2size(to_type); + } + + if (pI->get_opcode() == TEX_OP) { + inst.set_addr(lane_id, last_eaddr() ); + assert( inst.space == last_space() ); + insn_data_size = get_tex_datasize(pI, this); // texture obtain its data granularity from the texture info + } + + // Output register information to file and stdout + if( config.get_ptx_inst_debug_to_file()!=0 && + (config.get_ptx_inst_debug_thread_uid()==0||config.get_ptx_inst_debug_thread_uid()==get_uid()) ) { + dump_modifiedregs(m_gpu->get_ptx_inst_debug_file()); + dump_regs(m_gpu->get_ptx_inst_debug_file()); + } + + if ( g_debug_execution >= 6 ) { + if ( ptx_debug_exec_dump_cond<6>(get_uid(), pc) ) + dump_modifiedregs(stdout); + } + if ( g_debug_execution >= 10 ) { + if ( ptx_debug_exec_dump_cond<10>(get_uid(), pc) ) + dump_regs(stdout); + } + update_pc(); + g_ptx_sim_num_insn++; + + //not using it with functional simulation mode + if(!(this->m_functionalSimulationMode)) + ptx_file_line_stats_add_exec_count(pI); + + if ( gpgpu_ptx_instruction_classification ) { + init_inst_classification_stat(); + unsigned space_type=0; + switch ( pI->get_space().get_type() ) { + case global_space: space_type = 10; break; + case local_space: space_type = 11; break; + case tex_space: space_type = 12; break; + case surf_space: space_type = 13; break; + case param_space_kernel: + case param_space_local: + space_type = 14; break; + case shared_space: space_type = 15; break; + case const_space: space_type = 16; break; + default: + space_type = 0 ; + break; + } + StatAddSample( g_inst_classification_stat[g_ptx_kernel_count], op_classification); + if (space_type) StatAddSample( g_inst_classification_stat[g_ptx_kernel_count], ( int )space_type); + StatAddSample( g_inst_op_classification_stat[g_ptx_kernel_count], (int) pI->get_opcode() ); + } + if ( (g_ptx_sim_num_insn % 100000) == 0 ) { + dim3 ctaid = get_ctaid(); + dim3 tid = get_tid(); + printf("GPGPU-Sim PTX: %u instructions simulated : ctaid=(%u,%u,%u) tid=(%u,%u,%u)\n", + g_ptx_sim_num_insn, ctaid.x,ctaid.y,ctaid.z,tid.x,tid.y,tid.z ); + fflush(stdout); + } + + // "Return values" + if(!skip) { + inst.space = insn_space; + inst.set_addr(lane_id, insn_memaddr); + inst.data_size = insn_data_size; // simpleAtomicIntrinsics + assert( inst.memory_op == insn_memory_op ); + } + + } catch ( int x ) { + printf("GPGPU-Sim PTX: ERROR (%d) executing intruction (%s:%u)\n", x, pI->source_file(), pI->source_line() ); + printf("GPGPU-Sim PTX: '%s'\n", pI->get_source() ); + abort(); + } + +} + +void set_param_gpgpu_num_shaders(int num_shaders) +{ + gpgpu_param_num_shaders = num_shaders; +} + +const struct gpgpu_ptx_sim_info* ptx_sim_kernel_info(const function_info *kernel) +{ + return kernel->get_kernel_info(); +} + +const warp_inst_t *ptx_fetch_inst( address_type pc ) +{ + return function_info::pc_to_instruction(pc); +} + +unsigned ptx_sim_init_thread( kernel_info_t &kernel, + ptx_thread_info** thread_info, + int sid, + unsigned tid, + unsigned threads_left, + unsigned num_threads, + core_t *core, + unsigned hw_cta_id, + unsigned hw_warp_id, + gpgpu_t *gpu, + bool isInFunctionalSimulationMode) +{ + std::list &active_threads = kernel.active_threads(); + + static std::map shared_memory_lookup; + static std::map ptx_cta_lookup; + static std::map ptx_warp_lookup; + static std::map > local_memory_lookup; + + if ( *thread_info != NULL ) { + ptx_thread_info *thd = *thread_info; + assert( thd->is_done() ); + if ( g_debug_execution==-1 ) { + dim3 ctaid = thd->get_ctaid(); + dim3 t = thd->get_tid(); + printf("GPGPU-Sim PTX simulator: thread exiting ctaid=(%u,%u,%u) tid=(%u,%u,%u) uid=%u\n", + ctaid.x,ctaid.y,ctaid.z,t.x,t.y,t.z, thd->get_uid() ); + fflush(stdout); + } + thd->m_cta_info->register_deleted_thread(thd); + delete thd; + *thread_info = NULL; + } + + if ( !active_threads.empty() ) { + assert( active_threads.size() <= threads_left ); + ptx_thread_info *thd = active_threads.front(); + active_threads.pop_front(); + *thread_info = thd; + thd->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, isInFunctionalSimulationMode ); + return 1; + } + + if ( kernel.no_more_ctas_to_run() ) { + return 0; //finished! + } + + if ( threads_left < kernel.threads_per_cta() ) { + return 0; + } + + if ( g_debug_execution==-1 ) { + printf("GPGPU-Sim PTX simulator: STARTING THREAD ALLOCATION --> \n"); + fflush(stdout); + } + + //initializing new CTA + ptx_cta_info *cta_info = NULL; + memory_space *shared_mem = NULL; + + unsigned cta_size = kernel.threads_per_cta(); + unsigned max_cta_per_sm = num_threads/cta_size; // e.g., 256 / 48 = 5 + assert( max_cta_per_sm > 0 ); + + //unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid; + unsigned sm_idx = hw_cta_id*gpgpu_param_num_shaders + sid; + + if ( shared_memory_lookup.find(sm_idx) == shared_memory_lookup.end() ) { + if ( g_debug_execution >= 1 ) { + printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", + sm_idx, sid, max_cta_per_sm ); + } + char buf[512]; + snprintf(buf,512,"shared_%u", sid); + shared_mem = new memory_space_impl<16*1024>(buf,4); + shared_memory_lookup[sm_idx] = shared_mem; + cta_info = new ptx_cta_info(sm_idx); + ptx_cta_lookup[sm_idx] = cta_info; + } else { + if ( g_debug_execution >= 1 ) { + printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", + sm_idx, sid, max_cta_per_sm ); + } + shared_mem = shared_memory_lookup[sm_idx]; + cta_info = ptx_cta_lookup[sm_idx]; + cta_info->check_cta_thread_status_and_reset(); + } + + std::map &local_mem_lookup = local_memory_lookup[sid]; + while( kernel.more_threads_in_cta() ) { + dim3 ctaid3d = kernel.get_next_cta_id(); + unsigned new_tid = kernel.get_next_thread_id(); + dim3 tid3d = kernel.get_next_thread_id_3d(); + kernel.increment_thread_id(); + new_tid += tid; + ptx_thread_info *thd = new ptx_thread_info(kernel); + + ptx_warp_info *warp_info = NULL; + if ( ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end() ) { + warp_info = new ptx_warp_info(); + ptx_warp_lookup[hw_warp_id] = warp_info; + } else { + warp_info = ptx_warp_lookup[hw_warp_id]; + } + thd->m_warp_info = warp_info; + + memory_space *local_mem = NULL; + std::map::iterator l = local_mem_lookup.find(new_tid); + if ( l != local_mem_lookup.end() ) { + local_mem = l->second; + } else { + char buf[512]; + snprintf(buf,512,"local_%u_%u", sid, new_tid); + local_mem = new memory_space_impl<32>(buf,32); + local_mem_lookup[new_tid] = local_mem; + } + thd->set_info(kernel.entry()); + thd->set_nctaid(kernel.get_grid_dim()); + thd->set_ntid(kernel.get_cta_dim()); + thd->set_ctaid(ctaid3d); + thd->set_tid(tid3d); + if( kernel.entry()->get_ptx_version().extensions() ) + thd->cpy_tid_to_reg(tid3d); + thd->set_valid(); + thd->m_shared_mem = shared_mem; + function_info *finfo = thd->func_info(); + symbol_table *st = finfo->get_symtab(); + thd->func_info()->param_to_shared(thd->m_shared_mem,st); + thd->m_cta_info = cta_info; + cta_info->add_thread(thd); + thd->m_local_mem = local_mem; + if ( g_debug_execution==-1 ) { + printf("GPGPU-Sim PTX simulator: allocating thread ctaid=(%u,%u,%u) tid=(%u,%u,%u) @ 0x%Lx\n", + ctaid3d.x,ctaid3d.y,ctaid3d.z,tid3d.x,tid3d.y,tid3d.z, (unsigned long long)thd ); + fflush(stdout); + } + active_threads.push_back(thd); + } + if ( g_debug_execution==-1 ) { + printf("GPGPU-Sim PTX simulator: <-- FINISHING THREAD ALLOCATION\n"); + fflush(stdout); + } + + kernel.increment_cta_id(); + + assert( active_threads.size() <= threads_left ); + *thread_info = active_threads.front(); + (*thread_info)->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid,isInFunctionalSimulationMode ); + active_threads.pop_front(); + return 1; +} + +size_t get_kernel_code_size( class function_info *entry ) +{ + return entry->get_function_size(); +} + + +kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, + gpgpu_ptx_sim_arg_list_t args, + struct dim3 gridDim, + struct dim3 blockDim, + gpgpu_t *gpu ) +{ + kernel_info_t *result = new kernel_info_t(gridDim,blockDim,entry); + unsigned argcount=args.size(); + unsigned argn=1; + for( gpgpu_ptx_sim_arg_list_t::iterator a = args.begin(); a != args.end(); a++ ) { + entry->add_param_data(argcount-argn,&(*a)); + argn++; + } + entry->finalize(result->get_param_memory()); + g_ptx_kernel_count++; + fflush(stdout); + + return result; +} + +#include "../../version" +#include "detailed_version" + +void print_splash() +{ + static int splash_printed=0; + if ( !splash_printed ) { + fprintf(stdout, "\n\n *** %s [build %s] ***\n\n\n", g_gpgpusim_version_string, g_gpgpusim_build_string ); + splash_printed=1; + } +} + +std::map g_const_name_lookup; // indexed by hostVar +std::map g_global_name_lookup; // indexed by hostVar +std::set g_globals; +std::set g_constants; + +void gpgpu_ptx_sim_register_const_variable(void *hostVar, const char *deviceName, size_t size ) +{ + printf("GPGPU-Sim PTX registering constant %s (%zu bytes) to name mapping\n", deviceName, size ); + g_const_name_lookup[hostVar] = deviceName; +} + +void gpgpu_ptx_sim_register_global_variable(void *hostVar, const char *deviceName, size_t size ) +{ + printf("GPGPU-Sim PTX registering global %s hostVar to name mapping\n", deviceName ); + g_global_name_lookup[hostVar] = deviceName; +} + +void gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to, gpgpu_t *gpu ) +{ + printf("GPGPU-Sim PTX: starting gpgpu_ptx_sim_memcpy_symbol with hostVar 0x%p\n", hostVar); + bool found_sym = false; + memory_space_t mem_region = undefined_space; + std::string sym_name; + + std::map::iterator c=g_const_name_lookup.find(hostVar); + if ( c!=g_const_name_lookup.end() ) { + found_sym = true; + sym_name = c->second; + mem_region = const_space; + } + std::map::iterator g=g_global_name_lookup.find(hostVar); + if ( g!=g_global_name_lookup.end() ) { + if ( found_sym ) { + printf("Execution error: PTX symbol \"%s\" w/ hostVar=0x%Lx is declared both const and global?\n", + sym_name.c_str(), (unsigned long long)hostVar ); + abort(); + } + found_sym = true; + sym_name = g->second; + mem_region = global_space; + } + if( g_globals.find(hostVar) != g_globals.end() ) { + found_sym = true; + sym_name = hostVar; + mem_region = global_space; + } + if( g_constants.find(hostVar) != g_constants.end() ) { + found_sym = true; + sym_name = hostVar; + mem_region = const_space; + } + + if ( !found_sym ) { + printf("Execution error: No information for PTX symbol w/ hostVar=0x%Lx\n", (unsigned long long)hostVar ); + abort(); + } else printf("GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: Found PTX symbol w/ hostVar=0x%Lx\n", (unsigned long long)hostVar ); + const char *mem_name = NULL; + memory_space *mem = NULL; + + std::map::iterator st = g_sym_name_to_symbol_table.find(sym_name.c_str()); + assert( st != g_sym_name_to_symbol_table.end() ); + symbol_table *symtab = st->second; + + symbol *sym = symtab->lookup(sym_name.c_str()); + assert(sym); + unsigned dst = sym->get_address() + offset; + switch (mem_region.get_type()) { + case const_space: + mem = gpu->get_global_memory(); + mem_name = "const"; + break; + case global_space: + mem = gpu->get_global_memory(); + mem_name = "global"; + break; + default: + abort(); + } + printf("GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: copying %s memory %zu bytes %s symbol %s+%zu @0x%x ...\n", + mem_name, count, (to?" to ":"from"), sym_name.c_str(), offset, dst ); + for ( unsigned n=0; n < count; n++ ) { + if( to ) mem->write(dst+n,1,((char*)src)+n,NULL,NULL); + else mem->read(dst+n,1,((char*)src)+n); + } + fflush(stdout); +} + +int g_ptx_sim_mode; // if non-zero run functional simulation only (i.e., no notion of a clock cycle) + +extern int ptx_debug; + +bool g_cuda_launch_blocking = false; + +void read_sim_environment_variables() +{ + ptx_debug = 0; + g_debug_execution = 0; + g_interactive_debugger_enabled = false; + + char *mode = getenv("PTX_SIM_MODE_FUNC"); + if ( mode ) + sscanf(mode,"%u", &g_ptx_sim_mode); + printf("GPGPU-Sim PTX: simulation mode %d (can change with PTX_SIM_MODE_FUNC environment variable:\n", g_ptx_sim_mode); + printf(" 1=functional simulation only, 0=detailed performance simulator)\n"); + char *dbg_inter = getenv("GPGPUSIM_DEBUG"); + if ( dbg_inter && strlen(dbg_inter) ) { + printf("GPGPU-Sim PTX: enabling interactive debugger\n"); + fflush(stdout); + g_interactive_debugger_enabled = true; + } + char *dbg_level = getenv("PTX_SIM_DEBUG"); + if ( dbg_level && strlen(dbg_level) ) { + printf("GPGPU-Sim PTX: setting debug level to %s\n", dbg_level ); + fflush(stdout); + sscanf(dbg_level,"%d", &g_debug_execution); + } + char *dbg_thread = getenv("PTX_SIM_DEBUG_THREAD_UID"); + if ( dbg_thread && strlen(dbg_thread) ) { + printf("GPGPU-Sim PTX: printing debug information for thread uid %s\n", dbg_thread ); + fflush(stdout); + sscanf(dbg_thread,"%d", &g_debug_thread_uid); + } + char *dbg_pc = getenv("PTX_SIM_DEBUG_PC"); + if ( dbg_pc && strlen(dbg_pc) ) { + printf("GPGPU-Sim PTX: printing debug information for instruction with PC = %s\n", dbg_pc ); + fflush(stdout); + sscanf(dbg_pc,"%d", &g_debug_pc); + } + +#if CUDART_VERSION > 1010 + g_override_embedded_ptx = false; + char *usefile = getenv("PTX_SIM_USE_PTX_FILE"); + if (usefile && strlen(usefile)) { + printf("GPGPU-Sim PTX: overriding embedded ptx with ptx file (PTX_SIM_USE_PTX_FILE is set)\n"); + fflush(stdout); + g_override_embedded_ptx = true; + } + char *blocking = getenv("CUDA_LAUNCH_BLOCKING"); + if( blocking && !strcmp(blocking,"1") ) { + g_cuda_launch_blocking = true; + } +#else + g_cuda_launch_blocking = true; + g_override_embedded_ptx = true; +#endif + + if ( g_debug_execution >= 40 ) { + ptx_debug = 1; + } +} + +ptx_cta_info *g_func_cta_info = NULL; + +#define MAX(a,b) (((a)>(b))?(a):(b)) + +/*! +This function simulates the CUDA code functionally, it takes a kernel_info_t parameter +which holds the data for the CUDA kernel to be executed +!*/ +void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) +{ + printf("GPGPU-Sim: Performing Functional Simulation, executing kernel %s...\n",kernel.name().c_str()); + + //using a shader core object for book keeping, it is not needed but as most function built for performance simulation need it we use it here + extern gpgpu_sim *g_the_gpu; + //before we execute, we should do PDOM analysis for functional simulation scenario. + function_info *kernel_func_info = kernel.entry(); + if (kernel_func_info->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", kernel.name().c_str() ); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", kernel.name().c_str() ); + //kernel_func_info->do_pdom(); + kernel_func_info->set_pdom(); + } + + //we excute the kernel one CTA (Block) at the time, as synchronization functions work block wise + while(!kernel.no_more_ctas_to_run()){ + functionalCoreSim cta( + &kernel, + g_the_gpu, + g_the_gpu->getShaderCoreConfig()->warp_size + ); + cta.execute(); + +#if (CUDART_VERSION >= 5000) + launch_all_device_kernels(); +#endif + } + + //registering this kernel as done + + //openCL kernel simulation calls don't register the kernel so we don't register its exit + if(!openCL) { + extern stream_manager *g_stream_manager; + g_stream_manager->register_finished_kernel(kernel.get_uid()); + } + + //******PRINTING******* + printf( "GPGPU-Sim: Done functional simulation (%u instructions simulated).\n", g_ptx_sim_num_insn ); + if ( gpgpu_ptx_instruction_classification ) { + StatDisp( g_inst_classification_stat[g_ptx_kernel_count]); + StatDisp ( g_inst_op_classification_stat[g_ptx_kernel_count]); + } + + //time_t variables used to calculate the total simulation time + //the start time of simulation is hold by the global variable g_simulation_starttime + //g_simulation_starttime is initilized by gpgpu_ptx_sim_init_perf() in gpgpusim_entrypoint.cc upon starting gpgpu-sim + time_t end_time, elapsed_time, days, hrs, minutes, sec; + end_time = time((time_t *)NULL); + elapsed_time = MAX(end_time - g_simulation_starttime, 1); + + + //calculating and printing simulation time in terms of days, hours, minutes and seconds + days = elapsed_time/(3600*24); + hrs = elapsed_time/3600 - 24*days; + minutes = elapsed_time/60 - 60*(hrs + 24*days); + sec = elapsed_time - 60*(minutes + 60*(hrs + 24*days)); + + fflush(stderr); + printf("\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", + (unsigned)days, (unsigned)hrs, (unsigned)minutes, (unsigned)sec, (unsigned)elapsed_time ); + printf("gpgpu_simulation_rate = %u (inst/sec)\n", (unsigned)(g_ptx_sim_num_insn / elapsed_time) ); + fflush(stdout); +} + +void functionalCoreSim::initializeCTA() +{ + int ctaLiveThreads=0; + + for(int i=0; i< m_warp_count; i++){ + m_warpAtBarrier[i]=false; + m_liveThreadCount[i]=0; + } + for(int i=0; i< m_warp_count*m_warp_size;i++) + m_thread[i]=NULL; + + //get threads for a cta + for(unsigned i=0; ithreads_per_cta();i++) { + ptx_sim_init_thread(*m_kernel,&m_thread[i],0,i,m_kernel->threads_per_cta()-i,m_kernel->threads_per_cta(),this,0,i/m_warp_size,(gpgpu_t*)m_gpu, true); + assert(m_thread[i]!=NULL && !m_thread[i]->is_done()); + ctaLiveThreads++; + } + + for(int k=0;klaunch(m_thread[warpId*m_warp_size]->get_pc(),initialMask); + m_liveThreadCount[warpId]= liveThreadsCount; +} + +void functionalCoreSim::execute() + { + initializeCTA(); + + //start executing the CTA + while(true){ + bool someOneLive= false; + bool allAtBarrier = true; + for(unsigned i=0;i0) someOneLive=true; + if(!m_warpAtBarrier[i]&& m_liveThreadCount[i]>0) allAtBarrier = false; +} + +unsigned translate_pc_to_ptxlineno(unsigned pc) +{ + // this function assumes that the kernel fits inside a single PTX file + // function_info *pFunc = g_func_info; // assume that the current kernel is the one in query + const ptx_instruction *pInsn = function_info::pc_to_instruction(pc); + unsigned ptx_line_number = pInsn->source_line(); + + return ptx_line_number; +} + +// ptxinfo parser + +extern std::map get_duplicate(); + +int g_ptxinfo_error_detected; + +static char *g_ptxinfo_kname = NULL; +static struct gpgpu_ptx_sim_info g_ptxinfo; +static std::map g_duplicate; +static const char *g_last_dup_type; + +const char *get_ptxinfo_kname() +{ + return g_ptxinfo_kname; +} + +void print_ptxinfo() +{ + if(! get_ptxinfo_kname()){ + printf ("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", + g_ptxinfo.gmem, + g_ptxinfo.cmem); + } + if(get_ptxinfo_kname()){ + printf ("GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", + get_ptxinfo_kname(), + g_ptxinfo.regs, + g_ptxinfo.lmem, + g_ptxinfo.smem, + g_ptxinfo.cmem ); + } +} + + +struct gpgpu_ptx_sim_info get_ptxinfo() +{ + return g_ptxinfo; +} + +std::map get_duplicate() +{ + return g_duplicate; +} + +void ptxinfo_linenum( unsigned linenum ) +{ + g_duplicate[linenum] = g_last_dup_type; +} + +void ptxinfo_dup_type( const char *dup_type ) +{ + g_last_dup_type = dup_type; +} + +void ptxinfo_function(const char *fname ) +{ + clear_ptxinfo(); + g_ptxinfo_kname = strdup(fname); +} + +void ptxinfo_regs( unsigned nregs ) +{ + g_ptxinfo.regs=nregs; +} + +void ptxinfo_lmem( unsigned declared, unsigned system ) +{ + g_ptxinfo.lmem=declared+system; +} + +void ptxinfo_gmem( unsigned declared, unsigned system ) +{ + g_ptxinfo.gmem=declared+system; +} + +void ptxinfo_smem( unsigned declared, unsigned system ) +{ + g_ptxinfo.smem=declared+system; +} + +void ptxinfo_cmem( unsigned nbytes, unsigned bank ) +{ + g_ptxinfo.cmem+=nbytes; +} + +void clear_ptxinfo() +{ + free(g_ptxinfo_kname); + g_ptxinfo_kname=NULL; + g_ptxinfo.regs=0; + g_ptxinfo.lmem=0; + g_ptxinfo.smem=0; + g_ptxinfo.cmem=0; + g_ptxinfo.gmem=0; + g_ptxinfo.ptx_version=0; + g_ptxinfo.sm_target=0; +} + + +void ptxinfo_opencl_addinfo( std::map &kernels ) +{ + + if(! g_ptxinfo_kname) { + printf ("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", + g_ptxinfo.gmem, + g_ptxinfo.cmem); + clear_ptxinfo(); + return; + } + + if( !strcmp("__cuda_dummy_entry__",g_ptxinfo_kname) ) { + // this string produced by ptxas for empty ptx files (e.g., bandwidth test) + clear_ptxinfo(); + return; + } + std::map::iterator k=kernels.find(g_ptxinfo_kname); + if( k==kernels.end() ) { + printf ("GPGPU-Sim PTX: ERROR ** implementation for '%s' not found.\n", g_ptxinfo_kname ); + abort(); + } else { + printf ("GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", + g_ptxinfo_kname, + g_ptxinfo.regs, + g_ptxinfo.lmem, + g_ptxinfo.smem, + g_ptxinfo.cmem ); + function_info *finfo = k->second; + assert(finfo!=NULL); + finfo->set_kernel_info( g_ptxinfo ); + } + clear_ptxinfo(); +} + +struct rec_pts { + gpgpu_recon_t *s_kernel_recon_points; + int s_num_recon; +}; + +struct std::map g_rpts; + +struct rec_pts find_reconvergence_points( function_info *finfo ) +{ + rec_pts tmp; + std::map::iterator r=g_rpts.find(finfo); + + if( r==g_rpts.end() ) { + int num_recon = finfo->get_num_reconvergence_pairs(); + + gpgpu_recon_t *kernel_recon_points = (struct gpgpu_recon_t*) calloc(num_recon, sizeof(struct gpgpu_recon_t)); + finfo->get_reconvergence_pairs(kernel_recon_points); + printf("GPGPU-Sim PTX: reconvergence points for %s...\n", finfo->get_name().c_str() ); + for (int i=0;iprint_insn(); + printf("\n"); + printf("GPGPU-Sim PTX: immediate post dominator @ " ); + if( kernel_recon_points[i].target_inst ) + kernel_recon_points[i].target_inst->print_insn(); + printf("\n"); + } + printf("GPGPU-Sim PTX: ... end of reconvergence points for %s\n", finfo->get_name().c_str() ); + + tmp.s_kernel_recon_points = kernel_recon_points; + tmp.s_num_recon = num_recon; + g_rpts[finfo] = tmp; + } else { + tmp = r->second; + } + return tmp; +} + +address_type get_return_pc( void *thd ) +{ + // function call return + ptx_thread_info *the_thread = (ptx_thread_info*)thd; + assert( the_thread != NULL ); + return the_thread->get_return_PC(); +} + +address_type get_converge_point( address_type pc ) +{ + // the branch could encode the reconvergence point and/or a bit that indicates the + // reconvergence point is the return PC on the call stack in the case the branch has + // no immediate postdominator in the function (i.e., due to multiple return points). + + std::map::iterator f=g_pc_to_finfo.find(pc); + assert( f != g_pc_to_finfo.end() ); + function_info *finfo = f->second; + rec_pts tmp = find_reconvergence_points(finfo); + + int i=0; + for (; i < tmp.s_num_recon; ++i) { + if (tmp.s_kernel_recon_points[i].source_pc == pc) { + if( tmp.s_kernel_recon_points[i].target_pc == (unsigned) -2 ) { + return RECONVERGE_RETURN_PC; + } else { + return tmp.s_kernel_recon_points[i].target_pc; + } + } + } + return NO_BRANCH_DIVERGENCE; +} + +void functionalCoreSim::warp_exit( unsigned warp_id ) +{ + for(int i=0;im_cta_info->register_deleted_thread(m_thread[i]); + delete m_thread[i]; + } + } +} diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 5d97287..0025c52 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -1471,7 +1471,12 @@ void call_impl( const ptx_instruction *pI, ptx_thread_info *thread ) printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", target_func->get_name().c_str() ); } else { printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", target_func->get_name().c_str() ); - target_func->do_pdom(); + /* + * Some of the instructions like printf() gives the gpgpusim the wrong impression that it is a function call. + * As printf() doesnt have a body like functions do, doing pdom analysis for printf() causes a crash. + */ + if (target_func->get_function_size() >0) + target_func->do_pdom(); target_func->set_pdom(); } diff --git a/src/cuda-sim/instructions.cc~ b/src/cuda-sim/instructions.cc~ new file mode 100644 index 0000000..0e6f530 --- /dev/null +++ b/src/cuda-sim/instructions.cc~ @@ -0,0 +1,4517 @@ +// Copyright (c) 2009-2011, Tor M. Aamodt, Wilson W.L. Fung, Ali Bakhoda, +// Jimmy Kwa, George L. Yuan +// The University of British Columbia +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// Redistributions in binary form must reproduce the above copyright notice, this +// list of conditions and the following disclaimer in the documentation and/or +// other materials provided with the distribution. +// Neither the name of The University of British Columbia nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "instructions.h" +#include "ptx_ir.h" +#include "opcodes.h" +#include "ptx_sim.h" +#include "ptx.tab.h" +#include +#include +#include +#include "cuda-math.h" +#include "../abstract_hardware_model.h" +#include "ptx_loader.h" +#include "cuda_device_printf.h" +#include "../gpgpu-sim/gpu-sim.h" +#include "../gpgpu-sim/shader.h" + +//Jin: include device runtime for CDP +#include "cuda_device_runtime.h" + +#include + +unsigned ptx_instruction::g_num_ptx_inst_uid=0; + +const char *g_opcode_string[NUM_OPCODES] = { +#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR, +#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR, +#include "opcodes.def" +#undef OP_DEF +#undef OP_W_DEF +}; + +void inst_not_implemented( const ptx_instruction * pI ) ; +ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread); + +void sign_extend( ptx_reg_t &data, unsigned src_size, const operand_info &dst ); + +void ptx_thread_info::set_reg( const symbol *reg, const ptx_reg_t &value ) +{ + assert( reg != NULL ); + if( reg->name() == "_" ) return; + assert( !m_regs.empty() ); + assert( reg->uid() > 0 ); + m_regs.back()[ reg ] = value; + if (m_enable_debug_trace ) + m_debug_trace_regs_modified.back()[ reg ] = value; + m_last_set_operand_value = value; +} + +ptx_reg_t ptx_thread_info::get_reg( const symbol *reg ) +{ + static bool unfound_register_warned = false; + assert( reg != NULL ); + assert( !m_regs.empty() ); + reg_map_t::iterator regs_iter = m_regs.back().find(reg); + if (regs_iter == m_regs.back().end()) { + assert( reg->type()->get_key().is_reg() ); + const std::string &name = reg->name(); + unsigned call_uid = m_callstack.back().m_call_uid; + ptx_reg_t uninit_reg; + uninit_reg.u32 = 0x0; + set_reg(reg, uninit_reg); // give it a value since we are going to warn the user anyway + std::string file_loc = get_location(); + if( !unfound_register_warned ) { + printf("GPGPU-Sim PTX: WARNING (%s) ** reading undefined register \'%s\' (cuid:%u). Setting to 0X00000000. This is okay if you are simulating the native ISA" + "\n", + file_loc.c_str(), name.c_str(), call_uid ); + unfound_register_warned = true; + } + regs_iter = m_regs.back().find(reg); + } + if (m_enable_debug_trace ) + m_debug_trace_regs_read.back()[ reg ] = regs_iter->second; + return regs_iter->second; +} + +ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_info dstInfo, unsigned opType, ptx_thread_info *thread, int derefFlag ) +{ + ptx_reg_t result, tmp; + + + if(op.get_double_operand_type() == 0) { + if(((opType != BB128_TYPE) && (opType != BB64_TYPE) && (opType != FF64_TYPE)) || (op.get_addr_space() != undefined_space)) { + if ( op.is_reg() ) { + result = get_reg( op.get_symbol() ); + } else if ( op.is_builtin()) { + result.u32 = get_builtin( op.get_int(), op.get_addr_offset() ); + } else if(op.is_immediate_address()){ + result.u64 = op.get_addr_offset(); + } else if ( op.is_memory_operand() ) { + // a few options here... + const symbol *sym = op.get_symbol(); + const type_info *type = sym->type(); + const type_info_key &info = type->get_key(); + + if ( info.is_reg() ) { + const symbol *name = op.get_symbol(); + result.u64 = get_reg(name).u64 + op.get_addr_offset(); + } else if ( info.is_param_kernel() ) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if ( info.is_param_local() ) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if ( info.is_global() ) { + assert( op.get_addr_offset() == 0 ); + result.u64 = sym->get_address(); + } else if ( info.is_local() ) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if ( info.is_const() ) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if ( op.is_shared() ) { + result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); + } else { + const char *name = op.name().c_str(); + printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory operand type for %s\n", name ); + abort(); + } + + } else if ( op.is_literal() ) { + result = op.get_literal_value(); + } else if ( op.is_label() ) { + result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_shared() ) { + result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_const() ) { + result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_global() ) { + result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_local() ) { + result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_function_address() ) { + result.u64 = (size_t)op.get_symbol()->get_pc(); + } else { + const char *name = op.name().c_str(); + printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand type for %s\n", name ); + assert(0); + } + + if(op.get_operand_lohi() == 1) + result.u64 = result.u64 & 0xFFFF; + else if(op.get_operand_lohi() == 2) + result.u64 = (result.u64>>16) & 0xFFFF; + } else if (opType == BB128_TYPE) { + // b128 + result.u128.lowest = get_reg( op.vec_symbol(0) ).u32; + result.u128.low = get_reg( op.vec_symbol(1) ).u32; + result.u128.high = get_reg( op.vec_symbol(2) ).u32; + result.u128.highest = get_reg( op.vec_symbol(3) ).u32; + } else { + // bb64 or ff64 + result.bits.ls = get_reg( op.vec_symbol(0) ).u32; + result.bits.ms = get_reg( op.vec_symbol(1) ).u32; + } + } else if (op.get_double_operand_type() == 1) { + ptx_reg_t firstHalf, secondHalf; + firstHalf.u64 = get_reg( op.vec_symbol(0) ).u64; + secondHalf.u64 = get_reg( op.vec_symbol(1) ).u64; + if(op.get_operand_lohi() == 1) + secondHalf.u64 = secondHalf.u64 & 0xFFFF; + else if(op.get_operand_lohi() == 2) + secondHalf.u64 = (secondHalf.u64>>16) & 0xFFFF; + result.u64 = firstHalf.u64 + secondHalf.u64; + } else if (op.get_double_operand_type() == 2) { + // s[reg1 += reg2] + // reg1 is incremented after value is returned: the value returned is s[reg1] + ptx_reg_t firstHalf, secondHalf; + firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; + secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; + if(op.get_operand_lohi() == 1) + secondHalf.u64 = secondHalf.u64 & 0xFFFF; + else if(op.get_operand_lohi() == 2) + secondHalf.u64 = (secondHalf.u64>>16) & 0xFFFF; + result.u64 = firstHalf.u64; + firstHalf.u64 = firstHalf.u64 + secondHalf.u64; + set_reg(op.vec_symbol(0),firstHalf); + } else if (op.get_double_operand_type() == 3) { + // s[reg += immediate] + // reg is incremented after value is returned: the value returned is s[reg] + ptx_reg_t firstHalf; + firstHalf.u64 = get_reg(op.get_symbol()).u64; + result.u64 = firstHalf.u64; + firstHalf.u64 = firstHalf.u64 + op.get_addr_offset(); + set_reg(op.get_symbol(),firstHalf); + } + + ptx_reg_t finalResult; + memory_space *mem = NULL; + size_t size=0; + int t=0; + finalResult.u64=0; + + //complete other cases for reading from memory, such as reading from other const memory + if((op.get_addr_space() == global_space)&&(derefFlag)) { + // global memory - g[4], g[$r0] + mem = thread->get_global_memory(); + type_info_key::type_decode(opType,size,t); + mem->read(result.u32,size/8,&finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = global_space; + + if( opType == S16_TYPE || opType == S32_TYPE ) + sign_extend(finalResult,size,dstInfo); + } else if((op.get_addr_space() == shared_space)&&(derefFlag)) { + // shared memory - s[4], s[$r0] + mem = thread->m_shared_mem; + type_info_key::type_decode(opType,size,t); + mem->read(result.u32,size/8,&finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = shared_space; + + if( opType == S16_TYPE || opType == S32_TYPE ) + sign_extend(finalResult,size,dstInfo); + } else if((op.get_addr_space() == const_space)&&(derefFlag)) { + // const memory - ce0c1[4], ce0c1[$r0] + mem = thread->get_global_memory(); + type_info_key::type_decode(opType,size,t); + mem->read((result.u32 + op.get_const_mem_offset()),size/8,&finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = const_space; + if( opType == S16_TYPE || opType == S32_TYPE ) + sign_extend(finalResult,size,dstInfo); + } else if((op.get_addr_space() == local_space)&&(derefFlag)) { + // local memory - l0[4], l0[$r0] + mem = thread->m_local_mem; + type_info_key::type_decode(opType,size,t); + mem->read(result.u32,size/8,&finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = local_space; + if( opType == S16_TYPE || opType == S32_TYPE ) + sign_extend(finalResult,size,dstInfo); + } else { + finalResult = result; + } + + if((op.get_operand_neg() == true)&&(derefFlag)) { + switch( opType ) { + // Default to f32 for now, need to add support for others + case S8_TYPE: + case U8_TYPE: + case B8_TYPE: + finalResult.s8 = -finalResult.s8; + break; + case S16_TYPE: + case U16_TYPE: + case B16_TYPE: + finalResult.s16 = -finalResult.s16; + break; + case S32_TYPE: + case U32_TYPE: + case B32_TYPE: + finalResult.s32 = -finalResult.s32; + break; + case S64_TYPE: + case U64_TYPE: + case B64_TYPE: + finalResult.s64 = -finalResult.s64; + break; + case F16_TYPE: + finalResult.f16 = -finalResult.f16; + break; + case F32_TYPE: + finalResult.f32 = -finalResult.f32; + break; + case F64_TYPE: + case FF64_TYPE: + finalResult.f64 = -finalResult.f64; + break; + default: + assert(0); + } + + } + + return finalResult; + +} + +unsigned get_operand_nbits( const operand_info &op ) +{ + if ( op.is_reg() ) { + const symbol *sym = op.get_symbol(); + const type_info *typ = sym->type(); + type_info_key t = typ->get_key(); + switch( t.scalar_type() ) { + case PRED_TYPE: + return 1; + case B8_TYPE: case S8_TYPE: case U8_TYPE: + return 8; + case S16_TYPE: case U16_TYPE: case F16_TYPE: case B16_TYPE: + return 16; + case S32_TYPE: case U32_TYPE: case F32_TYPE: case B32_TYPE: + return 32; + case S64_TYPE: case U64_TYPE: case F64_TYPE: case B64_TYPE: + return 64; + default: + printf("ERROR: unknown register type\n"); + fflush(stdout); + abort(); + } + } else { + printf("ERROR: Need to implement get_operand_nbits() for currently unsupported operand_info type\n"); + fflush(stdout); + abort(); + } + + return 0; +} + +void ptx_thread_info::get_vector_operand_values( const operand_info &op, ptx_reg_t* ptx_regs, unsigned num_elements ) +{ + assert( op.is_vector() ); + assert( num_elements <= 4 ); // max 4 elements in a vector + + for (int idx = num_elements - 1; idx >= 0; --idx) { + const symbol *sym = NULL; + sym = op.vec_symbol(idx); + if( strcmp(sym->name().c_str(),"_") != 0) { + reg_map_t::iterator reg_iter = m_regs.back().find(sym); + assert( reg_iter != m_regs.back().end() ); + ptx_regs[idx] = reg_iter->second; + } + } +} + +void sign_extend( ptx_reg_t &data, unsigned src_size, const operand_info &dst ) +{ + if( !dst.is_reg() ) + return; + unsigned dst_size = get_operand_nbits( dst ); + if( src_size >= dst_size ) + return; + // src_size < dst_size + unsigned long long mask = 1; + mask <<= (src_size-1); + if( (mask & data.u64) == 0 ) { + // no need to sign extend + return; + } + // need to sign extend + mask = 1; + mask <<= dst_size-src_size; + mask -= 1; + mask <<= src_size; + data.u64 |= mask; +} + +void ptx_thread_info::set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI, int overflow, int carry ) +{ + thread->set_operand_value( dst, data, type, thread, pI ); + + if (dst.get_double_operand_type() == -2) + { + ptx_reg_t predValue; + + const symbol *sym = dst.vec_symbol(0); + predValue.u64 = (m_regs.back()[ sym ].u64) & ~(0x0C); + predValue.u64 |= ((overflow & 0x01)<<3); + predValue.u64 |= ((carry & 0x01)<<2); + + set_reg(sym,predValue); + } + else if (dst.get_double_operand_type() == 0) + { + //intentionally do nothing + } + else + { + printf("Unexpected double destination\n"); + assert(0); + } + +} + +void ptx_thread_info::set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI ) +{ + ptx_reg_t dstData; + memory_space *mem = NULL; + size_t size; + int t; + + type_info_key::type_decode(type,size,t); + + /*complete this section for other cases*/ + if(dst.get_addr_space() == undefined_space) + { + ptx_reg_t setValue; + setValue.u64 = data.u64; + + // Double destination in set instruction ($p0|$p1) - second is negation of first + if (dst.get_double_operand_type() == -1) + { + ptx_reg_t setValue2; + const symbol *name1 = dst.vec_symbol(0); + const symbol *name2 = dst.vec_symbol(1); + + if ( (type==F16_TYPE)||(type==F32_TYPE)||(type==F64_TYPE)||(type==FF64_TYPE) ) { + setValue2.f32 = (setValue.u64==0)?1.0f:0.0f; + } else { + setValue2.u32 = (setValue.u64==0)?0xFFFFFFFF:0; + } + + set_reg(name1,setValue); + set_reg(name2,setValue2); + } + + // Double destination in cvt,shr,mul,etc. instruction ($p0|$r4) - second register operand receives data, first predicate operand + // is set as $p0=($r4!=0) + // Also for Double destination in set instruction ($p0/$r1) + else if ((dst.get_double_operand_type() == -2)||(dst.get_double_operand_type() == -3)) + { + ptx_reg_t predValue; + const symbol *predName = dst.vec_symbol(0); + const symbol *regName = dst.vec_symbol(1); + predValue.u64 = 0; + + switch ( type ) { + case S8_TYPE: + if((setValue.s8 & 0x7F) == 0) + predValue.u64 |= 1; + break; + case S16_TYPE: + if((setValue.s16 & 0x7FFF) == 0) + predValue.u64 |= 1; + break; + case S32_TYPE: + if((setValue.s32 & 0x7FFFFFFF) == 0) + predValue.u64 |= 1; + break; + case S64_TYPE: + if((setValue.s64 & 0x7FFFFFFFFFFFFFFF) == 0) + predValue.u64 |= 1; + break; + case U8_TYPE: + case B8_TYPE: + if(setValue.u8 == 0) + predValue.u64 |= 1; + break; + case U16_TYPE: + case B16_TYPE: + if(setValue.u16 == 0) + predValue.u64 |= 1; + break; + case U32_TYPE: + case B32_TYPE: + if(setValue.u32 == 0) + predValue.u64 |= 1; + break; + case U64_TYPE: + case B64_TYPE: + if(setValue.u64 == 0) + predValue.u64 |= 1; + break; + case F16_TYPE: + if(setValue.f16 == 0) + predValue.u64 |= 1; + break; + case F32_TYPE: + if(setValue.f32 == 0) + predValue.u64 |= 1; + break; + case F64_TYPE: + case FF64_TYPE: + if(setValue.f64 == 0) + predValue.u64 |= 1; + break; + default: assert(0); break; + } + + + if ( (type==S8_TYPE)||(type==S16_TYPE)||(type==S32_TYPE)||(type==S64_TYPE)|| + (type==U8_TYPE)||(type==U16_TYPE)||(type==U32_TYPE)||(type==U64_TYPE)|| + (type==B8_TYPE)||(type==B16_TYPE)||(type==B32_TYPE)||(type==B64_TYPE)) { + if((setValue.u32 & (1<<(size-1))) != 0) + predValue.u64 |= 1<<1; + } + if ( type==F32_TYPE ) { + if(setValue.f32 < 0) + predValue.u64 |= 1<<1; + } + + if(dst.get_operand_lohi() == 1) + { + setValue.u64 = ((m_regs.back()[ regName ].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); + } + else if(dst.get_operand_lohi() == 2) + { + setValue.u64 = ((m_regs.back()[ regName ].u64) & (~(0xFFFF0000))) + ((data.u64<<16) & 0xFFFF0000); + } + + set_reg(predName,predValue); + set_reg(regName,setValue); + } + else if (type == BB128_TYPE) + { + //b128 stuff here. + ptx_reg_t setValue2, setValue3, setValue4; + setValue.u64 = 0; + setValue2.u64 = 0; + setValue3.u64 = 0; + setValue4.u64 = 0; + setValue.u32 = data.u128.lowest; + setValue2.u32 = data.u128.low; + setValue3.u32 = data.u128.high; + setValue4.u32 = data.u128.highest; + + const symbol *name1, *name2, *name3, *name4 = NULL; + + name1 = dst.vec_symbol(0); + name2 = dst.vec_symbol(1); + name3 = dst.vec_symbol(2); + name4 = dst.vec_symbol(3); + + set_reg(name1,setValue); + set_reg(name2,setValue2); + set_reg(name3,setValue3); + set_reg(name4,setValue4); + } + else if (type == BB64_TYPE || type == FF64_TYPE) + { + //ptxplus version of storing 64 bit values to registers stores to two adjacent registers + ptx_reg_t setValue2; + setValue.u32 = 0; + setValue2.u32 = 0; + + setValue.u32 = data.bits.ls; + setValue2.u32 = data.bits.ms; + + const symbol *name1, *name2 = NULL; + + name1 = dst.vec_symbol(0); + name2 = dst.vec_symbol(1); + + set_reg(name1,setValue); + set_reg(name2,setValue2); + } + else + { + if(dst.get_operand_lohi() == 1) + { + setValue.u64 = ((m_regs.back()[ dst.get_symbol() ].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); + } + else if(dst.get_operand_lohi() == 2) + { + setValue.u64 = ((m_regs.back()[ dst.get_symbol() ].u64) & (~(0xFFFF0000))) + ((data.u64<<16) & 0xFFFF0000); + } + set_reg(dst.get_symbol(),setValue); + } + } + + // global memory - g[4], g[$r0] + else if(dst.get_addr_space() == global_space) + { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->get_global_memory(); + type_info_key::type_decode(type,size,t); + + mem->write(dstData.u32,size/8,&data.u128,thread,pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = global_space; + } + + // shared memory - s[4], s[$r0] + else if(dst.get_addr_space() == shared_space) + { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->m_shared_mem; + type_info_key::type_decode(type,size,t); + + mem->write(dstData.u32,size/8,&data.u128,thread,pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = shared_space; + } + + // local memory - l0[4], l0[$r0] + else if(dst.get_addr_space() == local_space) + { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->m_local_mem; + type_info_key::type_decode(type,size,t); + + mem->write(dstData.u32,size/8,&data.u128,thread,pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = local_space; + } + + else + { + printf("Destination stores to unknown location."); + assert(0); + } + + +} + +void ptx_thread_info::set_vector_operand_values( const operand_info &dst, + const ptx_reg_t &data1, + const ptx_reg_t &data2, + const ptx_reg_t &data3, + const ptx_reg_t &data4 ) +{ + unsigned num_elements = dst.get_vect_nelem(); + if (num_elements > 0) { + set_reg(dst.vec_symbol(0), data1); + if (num_elements > 1) { + set_reg(dst.vec_symbol(1), data2); + if (num_elements > 2) { + set_reg(dst.vec_symbol(2), data3); + if (num_elements > 3) { + set_reg(dst.vec_symbol(3), data4); + } + } + } + } + + m_last_set_operand_value = data1; +} + +#define my_abs(a) (((a)<0)?(-a):(a)) + +#define MY_MAX_I(a,b) (a > b) ? a : b +#define MY_MAX_F(a,b) isNaN(a) ? b : isNaN(b) ? a : (a > b) ? a : b + +#define MY_MIN_I(a,b) (a < b) ? a : b +#define MY_MIN_F(a,b) isNaN(a) ? b : isNaN(b) ? a : (a < b) ? a : b + +#define MY_INC_I(a,b) (a >= b) ? 0 : a+1 +#define MY_DEC_I(a,b) ((a == 0) || (a > b)) ? b : a-1 + +#define MY_CAS_I(a,b,c) (a == b) ? c : a + +#define MY_EXCH(a,b) b + +void abs_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + + switch ( i_type ) { + case S16_TYPE: d.s16 = my_abs(a.s16); break; + case S32_TYPE: d.s32 = my_abs(a.s32); break; + case S64_TYPE: d.s64 = my_abs(a.s64); break; + case U16_TYPE: d.s16 = my_abs(a.u16); break; + case U32_TYPE: d.s32 = my_abs(a.u32); break; + case U64_TYPE: d.s64 = my_abs(a.u64); break; + case F32_TYPE: d.f32 = my_abs(a.f32); break; + case F64_TYPE: case FF64_TYPE: d.f64 = my_abs(a.f64); break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +void addp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + //PTXPlus add instruction with carry (carry is kept in a predicate) register + ptx_reg_t src1_data, src2_data, src3_data, data; + int overflow = 0; + int carry = 0; + + const operand_info &dst = pI->dst(); //get operand info of sources and destination + const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + + unsigned rounding_mode = pI->rounding_mode(); + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + + //performs addition. Sets carry and overflow if needed. + //src3_data.pred&0x4 is the carry flag + switch ( i_type ) { + case S8_TYPE: + data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF) + (src3_data.pred & 0x4); + if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) == 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } + carry = (data.u64 & 0x000000100)>>8; + break; + case S16_TYPE: + data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF) + (src3_data.pred & 0x4); + if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) == 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } + carry = (data.u64 & 0x000010000)>>16; + break; + case S32_TYPE: + data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF) + (src3_data.pred & 0x4); + if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) == 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } + carry = (data.u64 & 0x100000000)>>32; + break; + case S64_TYPE: + data.s64 = src1_data.s64 + src2_data.s64 + (src3_data.pred & 0x4); + break; + case U8_TYPE: + data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF) + (src3_data.pred & 0x4); + carry = (data.u64 & 0x100)>>8; + break; + case U16_TYPE: + data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF) + (src3_data.pred & 0x4); + carry = (data.u64 & 0x10000)>>16; + break; + case U32_TYPE: + data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF) + (src3_data.pred & 0x4); + carry = (data.u64 & 0x100000000)>>32; + break; + case U64_TYPE: + data.s64 = src1_data.s64 + src2_data.s64 + (src3_data.pred & 0x4); + break; + case F16_TYPE: assert(0); break; + case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; + case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; + default: assert(0); break; + } + fesetround( orig_rm ); + + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry ); +} + +void add_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + int overflow = 0; + int carry = 0; + + const operand_info &dst = pI->dst(); //get operand info of sources and destination + const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + unsigned rounding_mode = pI->rounding_mode(); + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + + //performs addition. Sets carry and overflow if needed. + switch ( i_type ) { + case S8_TYPE: + data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF); + if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) == 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } + carry = (data.u64 & 0x000000100)>>8; + break; + case S16_TYPE: + data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF); + if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) == 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } + carry = (data.u64 & 0x000010000)>>16; + break; + case S32_TYPE: + data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF); + if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) == 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } + carry = (data.u64 & 0x100000000)>>32; + break; + case S64_TYPE: + data.s64 = src1_data.s64 + src2_data.s64; + break; + case U8_TYPE: + data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF); + carry = (data.u64 & 0x100)>>8; + break; + case U16_TYPE: + data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF); + carry = (data.u64 & 0x10000)>>16; + break; + case U32_TYPE: + data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF); + carry = (data.u64 & 0x100000000)>>32; + break; + case U64_TYPE: + data.u64 = src1_data.u64 + src2_data.u64; + break; + case F16_TYPE: assert(0); break; + case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; + case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; + default: assert(0); break; + } + fesetround( orig_rm ); + + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry ); +} + +void addc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } + +void and_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) & ~(src2_data.pred)); + else + data.u64 = src1_data.u64 & src2_data.u64; + + thread->set_operand_value(dst,data, i_type, thread, pI); +} + +void andn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + switch ( i_type ) { + case B16_TYPE: src2_data.u16 = ~src2_data.u16; break; + case B32_TYPE: src2_data.u32 = ~src2_data.u32; break; + case B64_TYPE: src2_data.u64 = ~src2_data.u64; break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + data.u64 = src1_data.u64 & src2_data.u64; + + thread->set_operand_value(dst,data, i_type, thread, pI); +} + +void bar_callback( const inst_t* inst, ptx_thread_info* thread) +{ + unsigned ctaid = thread->get_cta_uid(); + unsigned barid = inst->bar_id; + unsigned value = thread->get_reduction_value(ctaid,barid); + const ptx_instruction *pI = dynamic_cast(inst); + const operand_info &dst = pI->dst(); + ptx_reg_t data; + data.u32 = value; + thread->set_operand_value(dst,value, U32_TYPE, thread, pI); +} + +void atom_callback( const inst_t* inst, ptx_thread_info* thread) +{ + const ptx_instruction *pI = dynamic_cast(inst); + + // "Decode" the output type + unsigned to_type = pI->get_type(); + size_t size; + int t; + type_info_key::type_decode(to_type, size, t); + + // Set up operand variables + ptx_reg_t data; // d + ptx_reg_t src1_data; // a + ptx_reg_t src2_data; // b + ptx_reg_t op_result; // temp variable to hold operation result + + bool data_ready = false; + + // Get operand info of sources and destination + const operand_info &dst = pI->dst(); // d + const operand_info &src1 = pI->src1(); // a + const operand_info &src2 = pI->src2(); // b + + // Get operand values + src1_data = thread->get_operand_value(src1, src1, to_type, thread, 1); // a + if (dst.get_symbol()->type()){ + src2_data = thread->get_operand_value(src2, dst, to_type, thread, 1); // b + } else { + //This is the case whent he first argument (dest) is '_' + src2_data = thread->get_operand_value(src2, src1, to_type, thread, 1); // b + } + + // Check state space + addr_t effective_address = src1_data.u64; + memory_space_t space = pI->get_space(); + if (space == undefined_space) { + // generic space - determine space via address + if( whichspace(effective_address) == global_space ) { + effective_address = generic_to_global(effective_address); + space = global_space; + } else if( whichspace(effective_address) == shared_space ) { + unsigned smid = thread->get_hw_sid(); + effective_address = generic_to_shared(smid,effective_address); + space = shared_space; + } else { + abort(); + } + } + assert( space == global_space || space == shared_space ); + + memory_space *mem = NULL; + if(space == global_space) + mem = thread->get_global_memory(); + else if(space == shared_space) + mem = thread->m_shared_mem; + else + abort(); + + // Copy value pointed to in operand 'a' into register 'd' + // (i.e. copy src1_data to dst) + mem->read(effective_address,size/8,&data.s64); + if (dst.get_symbol()->type()){ + thread->set_operand_value(dst, data, to_type, thread, pI); // Write value into register 'd' + } + + // Get the atomic operation to be performed + unsigned m_atomic_spec = pI->get_atomic(); + + switch ( m_atomic_spec ) { + // AND + case ATOMIC_AND: + { + + switch ( to_type ) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 & src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 & src2_data.s32; + data_ready = true; + break; + default: + printf("Execution error: type mismatch (%x) with instruction\natom.AND only accepts b32\n", to_type); + assert(0); + break; + } + + break; + } + // OR + case ATOMIC_OR: + { + + switch ( to_type ) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 | src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 | src2_data.s32; + data_ready = true; + break; + default: + printf("Execution error: type mismatch (%x) with instruction\natom.OR only accepts b32\n", to_type); + assert(0); + break; + } + + break; + } + // XOR + case ATOMIC_XOR: + { + + switch ( to_type ) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 ^ src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 ^ src2_data.s32; + data_ready = true; + break; + default: + printf("Execution error: type mismatch (%x) with instruction\natom.XOR only accepts b32\n", to_type); + assert(0); + break; + } + + break; + } + // CAS + case ATOMIC_CAS: + { + + ptx_reg_t src3_data; + const operand_info &src3 = pI->src3(); + src3_data = thread->get_operand_value(src3, dst, to_type, thread, 1); + + switch ( to_type ) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = MY_CAS_I(data.u32, src2_data.u32, src3_data.u32); + data_ready = true; + break; + case B64_TYPE: + case U64_TYPE: + op_result.u64 = MY_CAS_I(data.u64, src2_data.u64, src3_data.u64); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_CAS_I(data.s32, src2_data.s32, src3_data.s32); + data_ready = true; + break; + default: + printf("Execution error: type mismatch (%x) with instruction\natom.CAS only accepts b32 and b64\n", to_type); + assert(0); + break; + } + + break; + } + // EXCH + case ATOMIC_EXCH: + { + switch ( to_type ) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = MY_EXCH(data.u32, src2_data.u32); + data_ready = true; + break; + case B64_TYPE: + case U64_TYPE: + op_result.u64 = MY_EXCH(data.u64, src2_data.u64); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_EXCH(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf("Execution error: type mismatch (%x) with instruction\natom.EXCH only accepts b32\n", to_type); + assert(0); + break; + } + + break; + } + // ADD + case ATOMIC_ADD: + { + + switch ( to_type ) { + case U32_TYPE: + op_result.u32 = data.u32 + src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 + src2_data.s32; + data_ready = true; + break; + case U64_TYPE: + op_result.u64 = data.u64 + src2_data.u64; + data_ready = true; + break; + case F32_TYPE: + op_result.f32 = data.f32 + src2_data.f32; + data_ready = true; + break; + default: + printf("Execution error: type mismatch with instruction\natom.ADD only accepts u32, s32, u64, and f32\n"); + assert(0); + break; + } + + break; + } + // INC + case ATOMIC_INC: + { + switch ( to_type ) { + case U32_TYPE: + op_result.u32 = MY_INC_I(data.u32, src2_data.u32); + data_ready = true; + break; + default: + printf("Execution error: type mismatch with instruction\natom.INC only accepts u32 and s32\n"); + assert(0); + break; + } + + break; + } + // DEC + case ATOMIC_DEC: + { + switch ( to_type ) { + case U32_TYPE: + op_result.u32 = MY_DEC_I(data.u32, src2_data.u32); + data_ready = true; + break; + default: + printf("Execution error: type mismatch with instruction\natom.DEC only accepts u32 and s32\n"); + assert(0); + break; + } + + break; + } + // MIN + case ATOMIC_MIN: + { + switch ( to_type ) { + case U32_TYPE: + op_result.u32 = MY_MIN_I(data.u32, src2_data.u32); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_MIN_I(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf("Execution error: type mismatch with instruction\natom.MIN only accepts u32 and s32\n"); + assert(0); + break; + } + + break; + } + // MAX + case ATOMIC_MAX: + { + switch ( to_type ) { + case U32_TYPE: + op_result.u32 = MY_MAX_I(data.u32, src2_data.u32); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_MAX_I(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf("Execution error: type mismatch with instruction\natom.MAX only accepts u32 and s32\n"); + assert(0); + break; + } + + break; + } + // DEFAULT + default: + { + assert(0); + break; + } + } + + // Write operation result into memory + // (i.e. copy src1_data to dst) + if ( data_ready ) { + mem->write(effective_address,size/8,&op_result.s64,thread,pI); + } else { + printf("Execution error: data_ready not set\n"); + assert(0); + } +} + +// atom_impl will now result in a callback being called in mem_ctrl_pop (gpu-sim.c) +void atom_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + // SYNTAX + // atom.space.operation.type d, a, b[, c]; (now read in callback) + + // obtain memory space of the operation + memory_space_t space = pI->get_space(); + + // get the memory address + const operand_info &src1 = pI->src1(); + // const operand_info &dst = pI->dst(); // not needed for effective address calculation + unsigned i_type = pI->get_type(); + ptx_reg_t src1_data; + src1_data = thread->get_operand_value(src1, src1, i_type, thread, 1); + addr_t effective_address = src1_data.u64; + + addr_t effective_address_final; + + // handle generic memory space by converting it to global + if ( space == undefined_space ) { + if( whichspace(effective_address) == global_space ) { + effective_address_final = generic_to_global(effective_address); + space = global_space; + } else if( whichspace(effective_address) == shared_space ) { + unsigned smid = thread->get_hw_sid(); + effective_address_final = generic_to_shared(smid,effective_address); + space = shared_space; + } else { + abort(); + } + } else { + assert( space == global_space || space == shared_space ); + effective_address_final = effective_address; + } + + // Check state space + assert( space == global_space || space == shared_space ); + + thread->m_last_effective_address = effective_address_final; + thread->m_last_memory_space = space; + thread->m_last_dram_callback.function = atom_callback; + thread->m_last_dram_callback.instruction = pI; +} + +void bar_impl( const ptx_instruction *pIin, ptx_thread_info *thread ) +{ + ptx_instruction * pI = const_cast(pIin); + unsigned bar_op = pI->barrier_op(); + unsigned red_op = pI->get_atomic(); + unsigned ctaid = thread->get_cta_uid(); + + switch(bar_op){ + case SYNC_OPTION: + { + if(pI->get_num_operands()>1){ + const operand_info &op0 = pI->dst(); + const operand_info &op1 = pI->src1(); + ptx_reg_t op0_data; + ptx_reg_t op1_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + pI->set_bar_count(op1_data.u32); + }else{ + const operand_info &op0 = pI->dst(); + ptx_reg_t op0_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + } + break; + } + case ARRIVE_OPTION: + { + const operand_info &op0 = pI->dst(); + const operand_info &op1 = pI->src1(); + ptx_reg_t op0_data; + ptx_reg_t op1_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + pI->set_bar_count(op1_data.u32); + break; + } + case RED_OPTION: + { + if(pI->get_num_operands()>3){ + const operand_info &op1 = pI->src1(); + const operand_info &op2 = pI->src2(); + const operand_info &op3 = pI->src3(); + ptx_reg_t op1_data; + ptx_reg_t op2_data; + ptx_reg_t op3_data; + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + op2_data = thread->get_operand_value(op2, op2, U32_TYPE, thread, 1); + op3_data = thread->get_operand_value(op3, op3, PRED_TYPE, thread, 1); + op3_data.u32=!(op3_data.pred & 0x0001); + pI->set_bar_id(op1_data.u32); + pI->set_bar_count(op2_data.u32); + switch(red_op){ + case ATOMIC_POPC: + thread->popc_reduction(ctaid,op1_data.u32,op3_data.u32); + break; + case ATOMIC_AND: + thread->and_reduction(ctaid,op1_data.u32,op3_data.u32); + break; + case ATOMIC_OR: + thread->or_reduction(ctaid,op1_data.u32,op3_data.u32); + break; + default: + abort(); + break; + } + }else{ + const operand_info &op1 = pI->src1(); + const operand_info &op2 = pI->src2(); + ptx_reg_t op1_data; + ptx_reg_t op2_data; + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + op2_data = thread->get_operand_value(op2, op2, PRED_TYPE, thread, 1); + op2_data.u32=!(op2_data.pred & 0x0001); + pI->set_bar_id(op1_data.u32); + switch(red_op){ + case ATOMIC_POPC: + thread->popc_reduction(ctaid,op1_data.u32,op2_data.u32); + break; + case ATOMIC_AND: + thread->and_reduction(ctaid,op1_data.u32,op2_data.u32); + break; + case ATOMIC_OR: + thread->or_reduction(ctaid,op1_data.u32,op2_data.u32); + break; + default: + abort(); + break; + } + } + break; + } + default: + abort(); + break; + } + + thread->m_last_dram_callback.function = bar_callback; + thread->m_last_dram_callback.instruction = pIin; +} + +void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + unsigned i_type = pI->get_type(); + unsigned msb = (i_type == U32_TYPE || i_type == S32_TYPE) ? 31 : 63; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned pos = b.u32 & 0xFF; + unsigned len = c.u32 & 0xFF; + unsigned d = 0; + switch (i_type) + { + case U32_TYPE: + { + unsigned mask; + d = a.u32 >> pos; + mask = 0xFFFFFFFF >> (32 - len); + d &= mask; + break; + } + case U64_TYPE: + { + unsigned long mask; + d = a.u64 >> pos; + mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); + d &= mask; + break; + } + case S32_TYPE: + { + unsigned mask; + unsigned min = MY_MIN_I(pos + len - 1, msb); + unsigned sbit = len == 0 ? 0 : (a.s32 >> min) & 0x1; + d = a.s32 >> pos; + if (sbit > 0) + { + mask = 0xFFFFFFFF << len; + d |= mask; + } + else + { + mask = 0xFFFFFFFF >> (32 - len); + d &= mask; + } + break; + } + case S64_TYPE: + { + unsigned long mask; + unsigned min = MY_MIN_I(pos + len - 1, msb); + unsigned sbit = len == 0 ? 0 : (a.s64 >> min) & 0x1; + d = a.s64 >> pos; + if (sbit > 0) + { + mask = 0xFFFFFFFFFFFFFFFF << len; + d |= mask; + } + else + { + mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); + d &= mask; + } + break; + } + default: + printf("Operand type not supported for BFE instruction.\n"); + abort(); + return; + } + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +void bfi_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void bfind_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } + +void bra_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); + + thread->m_branch_taken = true; + thread->set_npc(target_pc); +} + +void brx_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); + + thread->m_branch_taken = true; + thread->set_npc(target_pc); +} + +void break_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &target = thread->pop_breakaddr(); + ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); + + thread->m_branch_taken = true; + thread->set_npc(target_pc); +} + +void breakaddr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &target = pI->dst(); + thread->push_breakaddr(target); + assert(pI->has_pred() == false); // pdom analysis cannot handle if this instruction is predicated +} + +void brev_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void brkpt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } + +void call_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + static unsigned call_uid_next = 1; + + const operand_info &target = pI->func_addr(); + assert( target.is_function_address() ); + const symbol *func_addr = target.get_symbol(); + function_info *target_func = func_addr->get_pc(); + if (target_func->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", target_func->get_name().c_str() ); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", target_func->get_name().c_str() ); + if (target_func->get_function_size() >0) + target_func->do_pdom(); + target_func->set_pdom(); + } + + // check that number of args and return match function requirements + if( pI->has_return() ^ target_func->has_return() ) { + printf("GPGPU-Sim PTX: Execution error - mismatch in number of return values between\n" + " call instruction and function declaration\n"); + abort(); + } + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + unsigned n_operands = pI->get_num_operands(); + + if( n_operands != (n_return+1+n_args) ) { + printf("GPGPU-Sim PTX: Execution error - mismatch in number of arguements between\n" + " call instruction and function declaration\n"); + abort(); + } + + // handle intrinsic functions + std::string fname = target_func->get_name(); + if( fname == "vprintf" ) { + gpgpusim_cuda_vprintf(pI, thread, target_func); + return; + } + +#if (CUDART_VERSION >= 5000) + //Jin: handle device runtime apis for CDP + else if(fname == "cudaGetParameterBufferV2") { + gpgpusim_cuda_getParameterBufferV2(pI, thread, target_func); + return; + } + else if(fname == "cudaLaunchDeviceV2") { + gpgpusim_cuda_launchDeviceV2(pI, thread, target_func); + return; + } + else if(fname == "cudaStreamCreateWithFlags") { + gpgpusim_cuda_streamCreateWithFlags(pI, thread, target_func); + return; + } +#endif + + // read source arguements into register specified in declaration of function + arg_buffer_list_t arg_values; + copy_args_into_buffer_list(pI, thread, target_func, arg_values); + + // record local for return value (we only support a single return value) + const symbol *return_var_src = NULL; + const symbol *return_var_dst = NULL; + if( target_func->has_return() ) { + return_var_dst = pI->dst().get_symbol(); + return_var_src = target_func->get_return_var(); + } + + gpgpu_sim *gpu = thread->get_gpu(); + unsigned callee_pc=0, callee_rpc=0; + if( gpu->simd_model() == POST_DOMINATOR ) { + thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(),&callee_pc,&callee_rpc); + assert( callee_pc == thread->get_pc() ); + } + + thread->callstack_push(callee_pc + pI->inst_size(), callee_rpc, return_var_src, return_var_dst, call_uid_next++); + + copy_buffer_list_into_frame(thread, arg_values); + + thread->set_npc(target_func); +} + +//Ptxplus version of call instruction. Jumps to a label not a different Kernel. +void callp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + + static unsigned call_uid_next = 1; + + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); + + const symbol *return_var_src = NULL; + const symbol *return_var_dst = NULL; + + gpgpu_sim *gpu = thread->get_gpu(); + unsigned callee_pc=0, callee_rpc=0; + if( gpu->simd_model() == POST_DOMINATOR ) { + thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(),&callee_pc,&callee_rpc); + assert( callee_pc == thread->get_pc() ); + } + + thread->callstack_push_plus(callee_pc + pI->inst_size(), callee_rpc, return_var_src, return_var_dst, call_uid_next++); + thread->set_npc(target_pc); +} + +void clz_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + int max; + unsigned long long mask; + d.u64 = 0; + + switch ( i_type ) { + case B32_TYPE: + max = 32; + mask = 0x80000000; + break; + case B64_TYPE: + max = 64; + mask = 0x8000000000000000; + break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + while ((d.u32 < max) && ((a.u64&mask) == 0) ) { + d.u32++; + a.u64 = a.u64 << 1; + } + + thread->set_operand_value(dst,d, B32_TYPE, thread, pI); +} + +void cnot_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch ( i_type ) { + case PRED_TYPE: d.pred = ((a.pred & 0x0001) == 0)?1:0; break; + case B16_TYPE: d.u16 = (a.u16 == 0)?1:0; break; + case B32_TYPE: d.u32 = (a.u32 == 0)?1:0; break; + case B64_TYPE: d.u64 = (a.u64 == 0)?1:0; break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +void cos_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + + switch ( i_type ) { + case F32_TYPE: + d.f32 = cos(a.f32); + break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +ptx_reg_t chop( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + switch ( to_width ) { + case 8: x.mask_and(0,0xFF); break; + case 16: x.mask_and(0,0xFFFF); break; + case 32: x.mask_and(0,0xFFFFFFFF); break; + case 64: break; + default: assert(0); + } + return x; +} + +ptx_reg_t sext( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + x=chop(x,0,from_width,0,rounding_mode,saturation_mode); + switch ( from_width ) { + case 8: if ( x.get_bit(7) ) x.mask_or(0xFFFFFFFF,0xFFFFFF00);break; + case 16:if ( x.get_bit(15) ) x.mask_or(0xFFFFFFFF,0xFFFF0000);break; + case 32: if ( x.get_bit(31) ) x.mask_or(0xFFFFFFFF,0x00000000);break; + case 64: break; + default: assert(0); + } + return x; +} + +// sign extend depending on the destination register size - hack to get SobelFilter working in CUDA 4.2 +ptx_reg_t sexd( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + x=chop(x,0,from_width,0,rounding_mode,saturation_mode); + switch ( to_width ) { + case 8: if ( x.get_bit(7) ) x.mask_or(0xFFFFFFFF,0xFFFFFF00);break; + case 16:if ( x.get_bit(15) ) x.mask_or(0xFFFFFFFF,0xFFFF0000);break; + case 32: if ( x.get_bit(31) ) x.mask_or(0xFFFFFFFF,0x00000000);break; + case 64: break; + default: assert(0); + } + return x; +} + +ptx_reg_t zext( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + return chop(x,0,from_width,0,rounding_mode,saturation_mode); +} + +int saturatei(int a, int max, int min) +{ + if (a > max) a = max; + else if (a < min) a = min; + return a; +} + +unsigned int saturatei(unsigned int a, unsigned int max) +{ + if (a > max) a = max; + return a; +} + +ptx_reg_t f2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + assert( from_width == 32); + + enum cuda_math::cudaRoundMode mode = cuda_math::cudaRoundZero; + switch (rounding_mode) { + case RZI_OPTION: mode = cuda_math::cudaRoundZero; break; + case RNI_OPTION: mode = cuda_math::cudaRoundNearest; break; + case RMI_OPTION: mode = cuda_math::cudaRoundMinInf; break; + case RPI_OPTION: mode = cuda_math::cudaRoundPosInf; break; + default: break; + } + + ptx_reg_t y; + if ( to_sign == 1 ) { // convert to 64-bit number first? + int tmp = cuda_math::float2int(x.f32, mode); + if ((x.u32 & 0x7f800000) == 0) + tmp = 0; // round denorm. FP to 0 + if (saturation_mode && to_width < 32) { + tmp = saturatei(tmp, (1< max) a = max; + else if (a < min) a = min; + return a; +} + +ptx_reg_t d2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + assert( from_width == 64); + + double tmp; + switch (rounding_mode) { + case RZI_OPTION: tmp = trunc(x.f64); break; + case RNI_OPTION: tmp = nearbyint(x.f64); break; + case RMI_OPTION: tmp = floor(x.f64); break; + case RPI_OPTION: tmp = ceil(x.f64); break; + default: tmp = x.f64; break; + } + + ptx_reg_t y; + if ( to_sign == 1 ) { + tmp = saturated2i(tmp, ((1<<(to_width - 1)) - 1), (1<<(to_width - 1)) ); + switch ( to_width ) { + case 8: y.s8 = (char)tmp; break; + case 16: y.s16 = (short)tmp; break; + case 32: y.s32 = (int)tmp; break; + case 64: y.s64 = (long long)tmp; break; + default: assert(0); break; + } + } else if ( to_sign == 0 ) { + tmp = saturated2i(tmp, ((1<<(to_width - 1)) - 1), 0); + switch ( to_width ) { + case 8: y.u8 = (unsigned char)tmp; break; + case 16: y.u16 = (unsigned short)tmp; break; + case 32: y.u32 = (unsigned int)tmp; break; + case 64: y.u64 = (unsigned long long)tmp; break; + default: assert(0); break; + } + } else { + switch ( to_width ) { + case 16: assert(0); break; + case 32: + y.f32 = x.f64; + break; + case 64: + y.f64 = x.f64; // should be handled by d2d + break; + default: assert(0); break; + } + } + return y; +} + +ptx_reg_t s2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + ptx_reg_t y; + + if (from_width < 64) { // 32-bit conversion + y = sext(x,from_width,32,0,rounding_mode,saturation_mode); + + switch ( to_width ) { + case 16: assert(0); break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: y.f32 = cuda_math::__int2float_rz(y.s32); break; + case RN_OPTION: y.f32 = cuda_math::__int2float_rn(y.s32); break; + case RM_OPTION: y.f32 = cuda_math::__int2float_rd(y.s32); break; + case RP_OPTION: y.f32 = cuda_math::__int2float_ru(y.s32); break; + default: break; + } + break; + case 64: y.f64 = y.s32; break; // no rounding needed + default: assert(0); break; + } + } else { + switch ( to_width ) { + case 16: assert(0); break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: y.f32 = cuda_math::__ll2float_rz(y.s64); break; + case RN_OPTION: y.f32 = cuda_math::__ll2float_rn(y.s64); break; + case RM_OPTION: y.f32 = cuda_math::__ll2float_rd(y.s64); break; + case RP_OPTION: y.f32 = cuda_math::__ll2float_ru(y.s64); break; + default: break; + } + break; + case 64: y.f64 = y.s64; break; // no internal implementation found + default: assert(0); break; + } + } + + // saturating an integer to 1 or 0? + return y; +} + +ptx_reg_t u2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + ptx_reg_t y; + + if (from_width < 64) { // 32-bit conversion + y = zext(x,from_width,32,0,rounding_mode,saturation_mode); + + switch ( to_width ) { + case 16: assert(0); break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: y.f32 = cuda_math::__uint2float_rz(y.u32); break; + case RN_OPTION: y.f32 = cuda_math::__uint2float_rn(y.u32); break; + case RM_OPTION: y.f32 = cuda_math::__uint2float_rd(y.u32); break; + case RP_OPTION: y.f32 = cuda_math::__uint2float_ru(y.u32); break; + default: break; + } + break; + case 64: y.f64 = y.u32; break; // no rounding needed + default: assert(0); break; + } + } else { + switch ( to_width ) { + case 16: assert(0); break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; + case RN_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; + case RM_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; + case RP_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; + default: break; + } + break; + case 64: y.f64 = y.u64; break; // no internal implementation found + default: assert(0); break; + } + } + + // saturating an integer to 1 or 0? + return y; +} + +ptx_reg_t f2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + ptx_reg_t y; + switch ( rounding_mode ) { + case RZI_OPTION: + y.f32 = truncf(x.f32); + break; + case RNI_OPTION: +#if CUDART_VERSION >= 3000 + y.f32 = nearbyintf(x.f32); +#else + y.f32 = cuda_math::__internal_nearbyintf(x.f32); +#endif + break; + case RMI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = floorf(x.f32); + } + break; + case RPI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = ceilf(x.f32); + } + break; + default: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = x.f32; + } + break; + } +#if CUDART_VERSION >= 3000 + if (isnanf(y.f32)) +#else + if (cuda_math::__cuda___isnanf(y.f32)) +#endif + { + y.u32 = 0x7fffffff; + } else if (saturation_mode) { + y.f32 = cuda_math::__saturatef(y.f32); + } + + return y; +} + +ptx_reg_t d2d( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) +{ + ptx_reg_t y; + switch ( rounding_mode ) { + case RZI_OPTION: + y.f64 = trunc(x.f64); + break; + case RNI_OPTION: +#if CUDART_VERSION >= 3000 + y.f64 = nearbyint(x.f64); +#else + y.f64 = cuda_math::__internal_nearbyintf(x.f64); +#endif + break; + case RMI_OPTION: + y.f64 = floor(x.f64); + break; + case RPI_OPTION: + y.f64 = ceil(x.f64); + break; + default: + y.f64 = x.f64; + break; + } + if (std::isnan(y.f64)) { + y.u64 = 0xfff8000000000000ull; + } else if (saturation_mode) { + y.f64 = cuda_math::__saturatef(y.f64); + } + return y; +} + +ptx_reg_t (*g_cvt_fn[11][11])( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode ) = { + { NULL, sext, sext, sext, NULL, sext, sext, sext, s2f, s2f, s2f}, + { chop, NULL, sext, sext, chop, NULL, sext, sext, s2f, s2f, s2f}, + { chop, sexd, NULL, sext, chop, chop, NULL, sext, s2f, s2f, s2f}, + { chop, chop, chop, NULL, chop, chop, chop, NULL, s2f, s2f, s2f}, + { NULL, zext, zext, zext, NULL, zext, zext, zext, u2f, u2f, u2f}, + { chop, NULL, zext, zext, chop, NULL, zext, zext, u2f, u2f, u2f}, + { chop, chop, NULL, zext, chop, chop, NULL, zext, u2f, u2f, u2f}, + { chop, chop, chop, NULL, chop, chop, chop, NULL, u2f, u2f, u2f}, + { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , NULL,f2x, f2x}, + { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x, f2f, f2x}, + { d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x, d2x, d2d} +}; + +void ptx_round(ptx_reg_t& data, int rounding_mode, int type) +{ + if (rounding_mode == RN_OPTION) { + return; + } + switch ( rounding_mode ) { + case RZI_OPTION: + switch ( type ) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); assert(0); break; + case F16_TYPE: assert(0); break; + case F32_TYPE: + data.f32 = truncf(data.f32); + break; + case F64_TYPE: + case FF64_TYPE: + if (data.f64 < 0) data.f64 = ceil(data.f64); //negative + else data.f64 = floor(data.f64); //positive + break; + default: assert(0); break; + } + break; + case RNI_OPTION: + switch ( type ) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); assert(0); break; + case F16_TYPE: assert(0); break; + case F32_TYPE: +#if CUDART_VERSION >= 3000 + data.f32 = nearbyintf(data.f32); +#else + data.f32 = cuda_math::__cuda_nearbyintf(data.f32); +#endif + break; + case F64_TYPE: case FF64_TYPE: data.f64 = round(data.f64); break; + default: assert(0); break; + } + break; + case RMI_OPTION: + switch ( type ) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); assert(0); break; + case F16_TYPE: assert(0); break; + case F32_TYPE: + data.f32 = floorf(data.f32); + break; + case F64_TYPE: case FF64_TYPE: data.f64 = floor(data.f64); break; + default: assert(0); break; + } + break; + case RPI_OPTION: + switch ( type ) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); assert(0); break; + case F16_TYPE: assert(0); break; + case F32_TYPE: data.f32 = ceilf(data.f32); break; + case F64_TYPE: case FF64_TYPE: data.f64 = ceil(data.f64); break; + default: assert(0); break; + } + break; + default: break; + } + + if (type == F32_TYPE) { +#if CUDART_VERSION >= 3000 + if (isnanf(data.f32)) +#else + if (cuda_math::__cuda___isnanf(data.f32)) +#endif + { + data.u32 = 0x7fffffff; + } + } + if ((type == F64_TYPE)||(type == FF64_TYPE)) { + if (std::isnan(data.f64)) { + data.u64 = 0xfff8000000000000ull; + } + } +} + +void ptx_saturate(ptx_reg_t& data, int saturation_mode, int type) +{ + if (!saturation_mode) { + return; + } + switch ( type ) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to clamp an integer to 1??\n"); assert(0); break; + case F16_TYPE: assert(0); break; + case F32_TYPE: + if (data.f32 > 1.0f) data.f32 = 1.0f; //negative + if (data.f32 < 0.0f) data.f32 = 0.0f; //positive + break; + case F64_TYPE: + case FF64_TYPE: + if (data.f64 > 1.0f) data.f64 = 1.0f; //negative + if (data.f64 < 0.0f) data.f64 = 0.0f; //positive + break; + default: assert(0); break; + } + +} + +void cvt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned to_type = pI->get_type(); + unsigned from_type = pI->get_type2(); + unsigned rounding_mode = pI->rounding_mode(); + unsigned saturation_mode = pI->saturation_mode(); + + if ( to_type == F16_TYPE || from_type == F16_TYPE ) + abort(); + + int to_sign, from_sign; + size_t from_width, to_width; + unsigned src_fmt = type_info_key::type_decode(from_type, from_width, from_sign); + unsigned dst_fmt = type_info_key::type_decode(to_type, to_width, to_sign); + + ptx_reg_t data = thread->get_operand_value(src1, dst, from_type, thread, 1); + + if(pI->is_neg()){ + + switch( from_type ) { + // Default to f32 for now, need to add support for others + case S8_TYPE: + case U8_TYPE: + case B8_TYPE: + data.s8 = -data.s8; + break; + case S16_TYPE: + case U16_TYPE: + case B16_TYPE: + data.s16 = -data.s16; + break; + case S32_TYPE: + case U32_TYPE: + case B32_TYPE: + data.s32 = -data.s32; + break; + case S64_TYPE: + case U64_TYPE: + case B64_TYPE: + data.s64 = -data.s64; + break; + case F16_TYPE: + data.f16 = -data.f16; + break; + case F32_TYPE: + data.f32 = -data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = -data.f64; + break; + default: + assert(0); + } + + } + + + if ( g_cvt_fn[src_fmt][dst_fmt] != NULL ) { + ptx_reg_t result = g_cvt_fn[src_fmt][dst_fmt](data,from_width,to_width,to_sign, rounding_mode, saturation_mode); + data = result; + } + + thread->set_operand_value(dst, data, to_type, thread, pI ); +} + +void cvta_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + memory_space_t space = pI->get_space(); + bool to_non_generic = pI->is_to(); + + unsigned i_type = pI->get_type(); + ptx_reg_t from_addr = thread->get_operand_value(src1,dst,i_type,thread,1); + addr_t from_addr_hw = (addr_t)from_addr.u64; + addr_t to_addr_hw = 0; + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); + + if( to_non_generic ) { + switch( space.get_type() ) { + case shared_space: to_addr_hw = generic_to_shared( smid, from_addr_hw ); break; + case local_space: to_addr_hw = generic_to_local( smid, hwtid, from_addr_hw ); break; + case global_space: to_addr_hw = generic_to_global(from_addr_hw ); break; + default: abort(); + } + } else { + switch( space.get_type() ) { + case shared_space: to_addr_hw = shared_to_generic( smid, from_addr_hw ); break; + case local_space: to_addr_hw = local_to_generic( smid, hwtid, from_addr_hw ) + + thread->get_local_mem_stack_pointer(); break; // add stack ptr here so that it can be passed as a pointer at function call + case global_space: to_addr_hw = global_to_generic( from_addr_hw ); break; + default: abort(); + } + } + + ptx_reg_t to_addr; + to_addr.u64 = to_addr_hw; + thread->set_reg(dst.get_symbol(),to_addr); +} + +void div_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + + switch ( i_type ) { + case S8_TYPE: + data.s8 = src1_data.s8 / src2_data.s8 ; break; + case S16_TYPE: + data.s16 = src1_data.s16 / src2_data.s16; break; + case S32_TYPE: + data.s32 = src1_data.s32 / src2_data.s32; break; + case S64_TYPE: + data.s64 = src1_data.s64 / src2_data.s64; break; + case U8_TYPE: + data.u8 = src1_data.u8 / src2_data.u8 ; break; + case U16_TYPE: + data.u16 = src1_data.u16 / src2_data.u16; break; + case U32_TYPE: + data.u32 = src1_data.u32 / src2_data.u32; break; + case U64_TYPE: + data.u64 = src1_data.u64 / src2_data.u64; break; + case B8_TYPE: + data.u8 = src1_data.u8 / src2_data.u8 ; break; + case B16_TYPE: + data.u16 = src1_data.u16 / src2_data.u16; break; + case B32_TYPE: + data.u32 = src1_data.u32 / src2_data.u32; break; + case B64_TYPE: + data.u64 = src1_data.u64 / src2_data.u64; break; + case F16_TYPE: assert(0); break; + case F32_TYPE: data.f32 = src1_data.f32 / src2_data.f32; break; + case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 / src2_data.f64; break; + default: assert(0); break; + } + thread->set_operand_value(dst,data, i_type, thread,pI); +} + +void ex2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + + switch ( i_type ) { + case F32_TYPE: + data.f32 = cuda_math::__powf(2.0, src1_data.f32); + break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst,data, i_type, thread,pI); +} + +void exit_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + thread->set_done(); + thread->exitCore(); + thread->registerExit(); +} + +void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry = false ); + +void fma_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + mad_def(pI,thread); +} + +void isspacep_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a; + bool t=false; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + memory_space_t space = pI->get_space(); + + a = thread->get_reg(src1.get_symbol()); + addr_t addr = (addr_t)a.u64; + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); + + switch( space.get_type() ) { + case shared_space: t = isspace_shared( smid, addr ); + case local_space: t = isspace_local( smid, hwtid, addr ); + case global_space: t = isspace_global( addr ); + default: abort(); + } + + ptx_reg_t p; + p.pred = t?1:0; + + thread->set_reg(dst.get_symbol(),p); +} + +void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand_info &op, memory_space *&mem, addr_t &addr) +{ + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); + + if( space == param_space_unclassified ) { + // need to op to determine whether it refers to a kernel param or local param + const symbol *s = op.get_symbol(); + const type_info *t = s->type(); + type_info_key ti = t->get_key(); + if( ti.is_param_kernel() ) + space = param_space_kernel; + else if( ti.is_param_local() ) { + space = param_space_local; + } else { + printf("GPGPU-Sim PTX: ERROR ** cannot resolve .param space for '%s'\n", s->name().c_str() ); + abort(); + } + } + switch ( space.get_type() ) { + case global_space: mem = thread->get_global_memory(); break; + case param_space_local: + case local_space: + mem = thread->m_local_mem; + addr += thread->get_local_mem_stack_pointer(); + break; + case tex_space: mem = thread->get_tex_memory(); break; + case surf_space: mem = thread->get_surf_memory(); break; + case param_space_kernel: mem = thread->get_param_memory(); break; + case shared_space: mem = thread->m_shared_mem; break; + case const_space: mem = thread->get_global_memory(); break; + case generic_space: + if( thread->get_ptx_version().ver() >= 2.0 ) { + // convert generic address to memory space address + space = whichspace(addr); + switch ( space.get_type() ) { + case global_space: mem = thread->get_global_memory(); addr = generic_to_global(addr); break; + case local_space: mem = thread->m_local_mem; addr = generic_to_local(smid,hwtid,addr); break; + case shared_space: mem = thread->m_shared_mem; addr = generic_to_shared(smid,addr); break; + default: abort(); + } + } else { + abort(); + } + break; + case param_space_unclassified: + case undefined_space: + default: + abort(); + } +} + +void ld_exec( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned type = pI->get_type(); + + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, type, thread, 1); + ptx_reg_t data; + memory_space_t space = pI->get_space(); + unsigned vector_spec = pI->get_vector(); + + memory_space *mem = NULL; + addr_t addr = src1_data.u32; + + decode_space(space,thread,src1,mem,addr); + + size_t size; + int t; + data.u64=0; + type_info_key::type_decode(type,size,t); + if (!vector_spec) { + mem->read(addr,size/8,&data.s64); + if( type == S16_TYPE || type == S32_TYPE ) + sign_extend(data,size,dst); + thread->set_operand_value(dst,data, type, thread, pI); + } else { + ptx_reg_t data1, data2, data3, data4; + mem->read(addr,size/8,&data1.s64); + mem->read(addr+size/8,size/8,&data2.s64); + if (vector_spec != V2_TYPE) { //either V3 or V4 + mem->read(addr+2*size/8,size/8,&data3.s64); + if (vector_spec != V3_TYPE) { //v4 + mem->read(addr+3*size/8,size/8,&data4.s64); + thread->set_vector_operand_values(dst,data1,data2,data3,data4); + } else //v3 + thread->set_vector_operand_values(dst,data1,data2,data3,data3); + } else //v2 + thread->set_vector_operand_values(dst,data1,data2,data2,data2); + } + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; +} + +void ld_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ld_exec(pI,thread); +} +void ldu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ld_exec(pI,thread); +} + +void lg2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + + switch ( i_type ) { + case F32_TYPE: + d.f32 = log(a.f32)/log(2); + break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +void mad24_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t d, t; + + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + + unsigned sat_mode = pI->saturation_mode(); + + assert( !pI->is_wide() ); + + switch ( i_type ) { + case S32_TYPE: + t.s64 = a.s32 * b.s32; + if ( pI->is_hi() ) { + d.s64 = (t.s64>>16) + c.s32; + if ( sat_mode ) { + if ( d.s64 > (int)0x7FFFFFFF ) + d.s64 = (int)0x7FFFFFFF; + else if ( d.s64 < (int)0x80000000 ) + d.s64 = (int)0x80000000; + } + } else if ( pI->is_lo() ) d.s64 = t.s32 + c.s32; + else assert(0); + break; + case U32_TYPE: + t.u64 = a.u32 * b.u32; + if ( pI->is_hi() ) d.u64 = (t.u64>>16) + c.u32; + else if ( pI->is_lo() ) d.u64 = t.u32 + c.u32; + else assert(0); + break; + default: + assert(0); + break; + } + + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +void mad_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + mad_def(pI, thread, false); +} + +void madp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + mad_def(pI, thread, true); +} + +void madc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + mad_def(pI, thread, true); +} + +void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t d, t; + + int carry=0; + int overflow=0; + + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + + // take the carry bit, it should be the 4th operand + ptx_reg_t carry_bit; + carry_bit.u64 = 0; + if (use_carry) { + const operand_info &carry = pI->operand_lookup(4); + carry_bit = thread->get_operand_value(carry, dst, PRED_TYPE, thread, 0); + carry_bit.pred &= 0x4; + carry_bit.pred >>=2; + } + + unsigned rounding_mode = pI->rounding_mode(); + + switch ( i_type ) { + case S16_TYPE: + t.s32 = a.s16 * b.s16; + if ( pI->is_wide() ) d.s32 = t.s32 + c.s32 + carry_bit.pred; + else if ( pI->is_hi() ) d.s16 = (t.s32>>16) + c.s16 + carry_bit.pred; + else if ( pI->is_lo() ) d.s16 = t.s16 + c.s16 + carry_bit.pred; + else assert(0); + carry = ((long long int)(t.s32 + c.s32 + carry_bit.pred)&0x100000000)>>32; + break; + case S32_TYPE: + t.s64 = a.s32 * b.s32; + if ( pI->is_wide() ) d.s64 = t.s64 + c.s64 + carry_bit.pred; + else if ( pI->is_hi() ) d.s32 = (t.s64>>32) + c.s32 + carry_bit.pred; + else if ( pI->is_lo() ) d.s32 = t.s32 + c.s32 + carry_bit.pred; + else assert(0); + break; + case S64_TYPE: + t.s64 = a.s64 * b.s64; + assert( !pI->is_wide() ); + assert( !pI->is_hi() ); + assert( use_carry == false); + if ( pI->is_lo() ) d.s64 = t.s64 + c.s64 + carry_bit.pred; + else assert(0); + break; + case U16_TYPE: + t.u32 = a.u16 * b.u16; + if ( pI->is_wide() ) d.u32 = t.u32 + c.u32 + carry_bit.pred; + else if ( pI->is_hi() ) d.u16 = (t.u32 + c.u16 + carry_bit.pred)>>16; + else if ( pI->is_lo() ) d.u16 = t.u16 + c.u16 + carry_bit.pred; + else assert(0); + carry = ((long long int)((long long int)t.u32 + c.u32 + carry_bit.pred)&0x100000000)>>32; + break; + case U32_TYPE: + t.u64 = a.u32 * b.u32; + if ( pI->is_wide() ) d.u64 = t.u64 + c.u64 + carry_bit.pred; + else if ( pI->is_hi() ) d.u32 = (t.u64 + c.u32 + carry_bit.pred)>>32; + else if ( pI->is_lo() ) d.u32 = t.u32 + c.u32 + carry_bit.pred; + else assert(0); + break; + case U64_TYPE: + t.u64 = a.u64 * b.u64; + assert( !pI->is_wide() ); + assert( !pI->is_hi() ); + assert( use_carry == false); + if ( pI->is_lo() ) d.u64 = t.u64 + c.u64 + carry_bit.pred; + else assert(0); + break; + case F16_TYPE: + assert(0); + break; + case F32_TYPE: { + assert( use_carry == false); + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + d.f32 = a.f32 * b.f32 + c.f32; + if ( pI->saturation_mode() ) { + if ( d.f32 < 0 ) d.f32 = 0; + else if ( d.f32 > 1.0f ) d.f32 = 1.0f; + } + fesetround( orig_rm ); + break; + } + case F64_TYPE: case FF64_TYPE: { + assert( use_carry == false); + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + d.f64 = a.f64 * b.f64 + c.f64; + if ( pI->saturation_mode() ) { + if ( d.f64 < 0 ) d.f64 = 0; + else if ( d.f64 > 1.0f ) d.f64 = 1.0; + } + fesetround( orig_rm ); + break; + } + default: + assert(0); + break; + } + thread->set_operand_value(dst, d, i_type, thread, pI, overflow, carry); +} + +bool isNaN(float x) +{ + return std::isnan(x); +} + +bool isNaN(double x) +{ + return std::isnan(x); +} + +void max_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + + + switch ( i_type ) { + case U16_TYPE: d.u16 = MY_MAX_I(a.u16,b.u16); break; + case U32_TYPE: d.u32 = MY_MAX_I(a.u32,b.u32); break; + case U64_TYPE: d.u64 = MY_MAX_I(a.u64,b.u64); break; + case S16_TYPE: d.s16 = MY_MAX_I(a.s16,b.s16); break; + case S32_TYPE: d.s32 = MY_MAX_I(a.s32,b.s32); break; + case S64_TYPE: d.s64 = MY_MAX_I(a.s64,b.s64); break; + case F32_TYPE: d.f32 = MY_MAX_F(a.f32,b.f32); break; + case F64_TYPE: case FF64_TYPE: d.f64 = MY_MAX_F(a.f64,b.f64); break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +void membar_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + // handled by timing simulator +} + +void min_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + + + switch ( i_type ) { + case U16_TYPE: d.u16 = MY_MIN_I(a.u16,b.u16); break; + case U32_TYPE: d.u32 = MY_MIN_I(a.u32,b.u32); break; + case U64_TYPE: d.u64 = MY_MIN_I(a.u64,b.u64); break; + case S16_TYPE: d.s16 = MY_MIN_I(a.s16,b.s16); break; + case S32_TYPE: d.s32 = MY_MIN_I(a.s32,b.s32); break; + case S64_TYPE: d.s64 = MY_MIN_I(a.s64,b.s64); break; + case F32_TYPE: d.f32 = MY_MIN_F(a.f32,b.f32); break; + case F64_TYPE: case FF64_TYPE: d.f64 = MY_MIN_F(a.f64,b.f64); break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +void mov_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + + if( (src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && (i_type != BB128_TYPE) && (i_type != FF64_TYPE) ) { + // pack or unpack operation + unsigned nbits_to_move; + ptx_reg_t tmp_bits; + + switch( pI->get_type() ) { + case B16_TYPE: nbits_to_move = 16; break; + case B32_TYPE: nbits_to_move = 32; break; + case B64_TYPE: nbits_to_move = 64; break; + default: printf("Execution error: mov pack/unpack with unsupported type qualifier\n"); assert(0); break; + } + + if( src1.is_vector() ) { + unsigned nelem = src1.get_vect_nelem(); + ptx_reg_t v[4]; + thread->get_vector_operand_values(src1, v, nelem ); + + unsigned bits_per_src_elem = nbits_to_move / nelem; + for( unsigned i=0; i < nelem; i++ ) { + switch(bits_per_src_elem) { + case 8: tmp_bits.u64 |= ((unsigned long long)(v[i].u8) << (8*i)); break; + case 16: tmp_bits.u64 |= ((unsigned long long)(v[i].u16) << (16*i)); break; + case 32: tmp_bits.u64 |= ((unsigned long long)(v[i].u32) << (32*i)); break; + default: printf("Execution error: mov pack/unpack with unsupported source/dst size ratio (src)\n"); assert(0); break; + } + } + } else { + data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch( pI->get_type() ) { + case B16_TYPE: tmp_bits.u16 = data.u16; break; + case B32_TYPE: tmp_bits.u32 = data.u32; break; + case B64_TYPE: tmp_bits.u64 = data.u64; break; + default: assert(0); break; + } + } + + if( dst.is_vector() ) { + unsigned nelem = dst.get_vect_nelem(); + ptx_reg_t v[4]; + unsigned bits_per_dst_elem = nbits_to_move / nelem; + for( unsigned i=0; i < nelem; i++ ) { + switch(bits_per_dst_elem) { + case 8: v[i].u8 = (tmp_bits.u64 >> (8*i)) & ((unsigned long long) 0xFF); break; + case 16: v[i].u16 = (tmp_bits.u64 >> (16*i)) & ((unsigned long long) 0xFFFF); break; + case 32: v[i].u32 = (tmp_bits.u64 >> (32*i)) & ((unsigned long long) 0xFFFFFFFF); break; + default: + printf("Execution error: mov pack/unpack with unsupported source/dst size ratio (dst)\n"); + assert(0); + break; + } + } + thread->set_vector_operand_values(dst,v[0],v[1],v[2],v[3]); + } else { + thread->set_operand_value(dst,tmp_bits, i_type, thread, pI); + } + } else if (i_type == PRED_TYPE and src1.is_literal() == true) { + // in ptx, literal input translate to predicate as 0 = false and 1 = true + // we have adopted the opposite to simplify implementation of zero flags in ptxplus + data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + ptx_reg_t finaldata; + finaldata.pred = (data.u32 == 0)? 1 : 0; // setting zero-flag in predicate + thread->set_operand_value(dst, finaldata, i_type, thread, pI); + } else { + + data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + thread->set_operand_value(dst, data, i_type, thread, pI); + + } +} + +void mul24_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + + //src1_data = srcOperandModifiers(src1_data, src1, dst, i_type, thread); + //src2_data = srcOperandModifiers(src2_data, src2, dst, i_type, thread); + + src1_data.mask_and(0,0x00FFFFFF); + src2_data.mask_and(0,0x00FFFFFF); + + switch ( i_type ) { + case S32_TYPE: + if( src1_data.get_bit(23) ) + src1_data.mask_or(0xFFFFFFFF,0xFF000000); + if( src2_data.get_bit(23) ) + src2_data.mask_or(0xFFFFFFFF,0xFF000000); + data.s64 = src1_data.s64 * src2_data.s64; + break; + case U32_TYPE: + data.u64 = src1_data.u64 * src2_data.u64; + break; + default: + printf("GPGPU-Sim PTX: Execution error - type mismatch with instruction\n"); + assert(0); + break; + } + + if ( pI->is_hi() ) { + data.u64 = data.u64 >> 16; + data.mask_and(0,0xFFFFFFFF); + } else if (pI->is_lo()) { + data.mask_and(0,0xFFFFFFFF); + } + + thread->set_operand_value(dst, data, i_type, thread, pI); +} + +void mul_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + ptx_reg_t d, t; + + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + + unsigned rounding_mode = pI->rounding_mode(); + + switch ( i_type ) { + case S16_TYPE: + t.s32 = ((int)a.s16) * ((int)b.s16); + if ( pI->is_wide() ) d.s32 = t.s32; + else if ( pI->is_hi() ) d.s16 = (t.s32>>16); + else if ( pI->is_lo() ) d.s16 = t.s16; + else assert(0); + break; + case S32_TYPE: + t.s64 = ((long long)a.s32) * ((long long)b.s32); + if ( pI->is_wide() ) d.s64 = t.s64; + else if ( pI->is_hi() ) d.s32 = (t.s64>>32); + else if ( pI->is_lo() ) d.s32 = t.s32; + else assert(0); + break; + case S64_TYPE: + t.s64 = a.s64 * b.s64; + assert( !pI->is_wide() ); + assert( !pI->is_hi() ); + if ( pI->is_lo() ) d.s64 = t.s64; + else assert(0); + break; + case U16_TYPE: + t.u32 = ((unsigned)a.u16) * ((unsigned)b.u16); + if ( pI->is_wide() ) d.u32 = t.u32; + else if ( pI->is_lo() ) d.u16 = t.u16; + else if ( pI->is_hi() ) d.u16 = (t.u32>>16); + else assert(0); + break; + case U32_TYPE: + t.u64 = ((unsigned long long)a.u32) * ((unsigned long long)b.u32); + if ( pI->is_wide() ) d.u64 = t.u64; + else if ( pI->is_lo() ) d.u32 = t.u32; + else if ( pI->is_hi() ) d.u32 = (t.u64>>32); + else assert(0); + break; + case U64_TYPE: + t.u64 = a.u64 * b.u64; + assert( !pI->is_wide() ); + assert( !pI->is_hi() ); + if ( pI->is_lo() ) d.u64 = t.u64; + else assert(0); + break; + case F16_TYPE: + assert(0); + break; + case F32_TYPE: { + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + + d.f32 = a.f32 * b.f32; + + if ( pI->saturation_mode() ) { + if ( d.f32 < 0 ) d.f32 = 0; + else if ( d.f32 > 1.0f ) d.f32 = 1.0f; + } + fesetround( orig_rm ); + break; + } + case F64_TYPE: case FF64_TYPE:{ + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + d.f64 = a.f64 * b.f64; + if ( pI->saturation_mode() ) { + if ( d.f64 < 0 ) d.f64 = 0; + else if ( d.f64 > 1.0f ) d.f64 = 1.0; + } + fesetround( orig_rm ); + break; + } + default: + assert(0); + break; + } + + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +void neg_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned to_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, to_type, thread, 1); + + + switch ( to_type ) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + data.s64 = 0 - src1_data.s64; break; // seems buggy, but not (just ignore higher bits) + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + assert(0); break; + case F16_TYPE: assert(0); break; + case F32_TYPE: data.f32 = 0.0f - src1_data.f32; break; + case F64_TYPE: case FF64_TYPE: data.f64 = 0.0f - src1_data.f64; break; + default: assert(0); break; + } + + thread->set_operand_value(dst,data, to_type, thread, pI); +} + +//nandn bitwise negates second operand then bitwise nands with the first operand +void nandn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = (~src1_data.pred & src2_data.pred); + else + data.u64 = ~(src1_data.u64 & ~src2_data.u64); + + thread->set_operand_value(dst,data, i_type, thread, pI); + +} + +//norn bitwise negates first operand then bitwise ands with the second operand +void norn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = ~(src1_data.pred & ~(src2_data.pred)); + else + data.u64 = ~(src1_data.u64) & src2_data.u64; + + thread->set_operand_value(dst,data, i_type, thread, pI); + +} + +void not_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + + switch ( i_type ) { + case PRED_TYPE: d.pred = (~(a.pred) & 0x000F); break; + case B16_TYPE: d.u16 = ~a.u16; break; + case B32_TYPE: d.u32 = ~a.u32; break; + case B64_TYPE: d.u64 = ~a.u64; break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +void or_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) | ~(src2_data.pred)); + else + data.u64 = src1_data.u64 | src2_data.u64; + + thread->set_operand_value(dst,data, i_type, thread, pI); +} + +void orn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) | (src2_data.pred)); + else + data.u64 = src1_data.u64 | ~src2_data.u64; + + thread->set_operand_value(dst,data, i_type, thread, pI); +} + +void pmevent_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void popc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src = pI->src1(); + + unsigned i_type = pI->get_type(); + src_data = thread->get_operand_value(src, dst, i_type, thread, 1); + + switch ( i_type ) { + case B32_TYPE: { + std::bitset<32> mask(src_data.u32); + data.u32 = mask.count(); + } break; + case B64_TYPE: { + std::bitset<64> mask(src_data.u64); + data.u32 = mask.count(); + } break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst,data, i_type, thread, pI); +} +void prefetch_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void prefetchu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void prmt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } + +void rcp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + + switch ( i_type ) { + case F32_TYPE: + data.f32 = 1.0f / src1_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = 1.0f / src1_data.f64; + break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst,data, i_type, thread, pI); +} + +void red_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } + +void rem_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + data.u64 = src1_data.u64 % src2_data.u64; + + thread->set_operand_value(dst,data, i_type, thread, pI); +} + +void ret_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + bool empty = thread->callstack_pop(); + if( empty ) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); + } +} + +//Ptxplus version of ret instruction. +void retp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + bool empty = thread->callstack_pop_plus(); + if( empty ) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); + } +} + +void rsqrt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + + switch ( i_type ) { + case F32_TYPE: + if ( a.f32 < 0 ) { + d.u64 = 0; + d.u64 = 0x7fc00000; // NaN + } else if ( a.f32 == 0 ) { + d.u64 = 0; + d.u32 = 0x7f800000; // Inf + } else + d.f32 = cuda_math::__internal_accurate_fdividef(1.0f, sqrtf(a.f32)); + break; + case F64_TYPE: + case FF64_TYPE: + if ( a.f32 < 0 ) { + d.u64 = 0; + d.u32 = 0x7fc00000; // NaN + float x = d.f32; + d.f64 = (double)x; + } else if ( a.f32 == 0 ) { + d.u64 = 0; + d.u32 = 0x7f800000; // Inf + float x = d.f32; + d.f64 = (double)x; + } else + d.f64 = 1.0 / sqrt(a.f64); + break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +#define SAD(d,a,b,c) d = c + ((adst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, i_type, thread, 1); + + + switch ( i_type ) { + case U16_TYPE: SAD(d.u16,a.u16,b.u16,c.u16); break; + case U32_TYPE: SAD(d.u32,a.u32,b.u32,c.u32); break; + case U64_TYPE: SAD(d.u64,a.u64,b.u64,c.u64); break; + case S16_TYPE: SAD(d.s16,a.s16,b.s16,c.s16); break; + case S32_TYPE: SAD(d.s32,a.s32,b.s32,c.s32); break; + case S64_TYPE: SAD(d.s64,a.s64,b.s64,c.s64); break; + case F32_TYPE: SAD(d.f32,a.f32,b.f32,c.f32); break; + case F64_TYPE: case FF64_TYPE: SAD(d.f64,a.f64,b.f64,c.f64); break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +void selp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + ptx_reg_t a, b, c, d; + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, i_type, thread, 1); + + //predicate value was changed so the lowest bit being set means the zero flag is set. + //As a result, the value of c.pred must be inverted to get proper behavior + d = (!(c.pred & 0x0001))?a:b; + + thread->set_operand_value(dst,d, PRED_TYPE, thread, pI); +} + +bool isFloat(int type) +{ + switch ( type ) { + case F16_TYPE: + case F32_TYPE: + case F64_TYPE: + case FF64_TYPE: + return true; + default: + return false; + } +} + +bool CmpOp( int type, ptx_reg_t a, ptx_reg_t b, unsigned cmpop ) +{ + bool t = false; + + switch ( type ) { + case B16_TYPE: + switch (cmpop) { + case EQ_OPTION: t = (a.u16 == b.u16); break; + case NE_OPTION: t = (a.u16 != b.u16); break; + default: + assert(0); + } + + case B32_TYPE: + switch (cmpop) { + case EQ_OPTION: t = (a.u32 == b.u32); break; + case NE_OPTION: t = (a.u32 != b.u32); break; + default: + assert(0); + } + case B64_TYPE: + switch (cmpop) { + case EQ_OPTION: t = (a.u64 == b.u64); break; + case NE_OPTION: t = (a.u64 != b.u64); break; + default: + assert(0); + } + break; + case S8_TYPE: + case S16_TYPE: + switch (cmpop) { + case EQ_OPTION: t = (a.s16 == b.s16); break; + case NE_OPTION: t = (a.s16 != b.s16); break; + case LT_OPTION: t = (a.s16 < b.s16); break; + case LE_OPTION: t = (a.s16 <= b.s16); break; + case GT_OPTION: t = (a.s16 > b.s16); break; + case GE_OPTION: t = (a.s16 >= b.s16); break; + default: + assert(0); + } + break; + case S32_TYPE: + switch (cmpop) { + case EQ_OPTION: t = (a.s32 == b.s32); break; + case NE_OPTION: t = (a.s32 != b.s32); break; + case LT_OPTION: t = (a.s32 < b.s32); break; + case LE_OPTION: t = (a.s32 <= b.s32); break; + case GT_OPTION: t = (a.s32 > b.s32); break; + case GE_OPTION: t = (a.s32 >= b.s32); break; + default: + assert(0); + } + break; + case S64_TYPE: + switch (cmpop) { + case EQ_OPTION: t = (a.s64 == b.s64); break; + case NE_OPTION: t = (a.s64 != b.s64); break; + case LT_OPTION: t = (a.s64 < b.s64); break; + case LE_OPTION: t = (a.s64 <= b.s64); break; + case GT_OPTION: t = (a.s64 > b.s64); break; + case GE_OPTION: t = (a.s64 >= b.s64); break; + default: + assert(0); + } + break; + case U8_TYPE: + case U16_TYPE: + switch (cmpop) { + case EQ_OPTION: t = (a.u16 == b.u16); break; + case NE_OPTION: t = (a.u16 != b.u16); break; + case LT_OPTION: t = (a.u16 < b.u16); break; + case LE_OPTION: t = (a.u16 <= b.u16); break; + case GT_OPTION: t = (a.u16 > b.u16); break; + case GE_OPTION: t = (a.u16 >= b.u16); break; + case LO_OPTION: t = (a.u16 < b.u16); break; + case LS_OPTION: t = (a.u16 <= b.u16); break; + case HI_OPTION: t = (a.u16 > b.u16); break; + case HS_OPTION: t = (a.u16 >= b.u16); break; + default: + assert(0); + } + break; + case U32_TYPE: + switch (cmpop) { + case EQ_OPTION: t = (a.u32 == b.u32); break; + case NE_OPTION: t = (a.u32 != b.u32); break; + case LT_OPTION: t = (a.u32 < b.u32); break; + case LE_OPTION: t = (a.u32 <= b.u32); break; + case GT_OPTION: t = (a.u32 > b.u32); break; + case GE_OPTION: t = (a.u32 >= b.u32); break; + case LO_OPTION: t = (a.u32 < b.u32); break; + case LS_OPTION: t = (a.u32 <= b.u32); break; + case HI_OPTION: t = (a.u32 > b.u32); break; + case HS_OPTION: t = (a.u32 >= b.u32); break; + default: + assert(0); + } + break; + case U64_TYPE: + switch (cmpop) { + case EQ_OPTION: t = (a.u64 == b.u64); break; + case NE_OPTION: t = (a.u64 != b.u64); break; + case LT_OPTION: t = (a.u64 < b.u64); break; + case LE_OPTION: t = (a.u64 <= b.u64); break; + case GT_OPTION: t = (a.u64 > b.u64); break; + case GE_OPTION: t = (a.u64 >= b.u64); break; + case LO_OPTION: t = (a.u64 < b.u64); break; + case LS_OPTION: t = (a.u64 <= b.u64); break; + case HI_OPTION: t = (a.u64 > b.u64); break; + case HS_OPTION: t = (a.u64 >= b.u64); break; + default: + assert(0); + } + break; + case F16_TYPE: assert(0); break; + case F32_TYPE: + switch (cmpop) { + case EQ_OPTION: t = (a.f32 == b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; + case NE_OPTION: t = (a.f32 != b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; + case LT_OPTION: t = (a.f32 < b.f32 ) && !isNaN(a.f32) && !isNaN(b.f32); break; + case LE_OPTION: t = (a.f32 <= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; + case GT_OPTION: t = (a.f32 > b.f32 ) && !isNaN(a.f32) && !isNaN(b.f32); break; + case GE_OPTION: t = (a.f32 >= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; + case EQU_OPTION: t = (a.f32 == b.f32) || isNaN(a.f32) || isNaN(b.f32); break; + case NEU_OPTION: t = (a.f32 != b.f32) || isNaN(a.f32) || isNaN(b.f32); break; + case LTU_OPTION: t = (a.f32 < b.f32 ) || isNaN(a.f32) || isNaN(b.f32); break; + case LEU_OPTION: t = (a.f32 <= b.f32) || isNaN(a.f32) || isNaN(b.f32); break; + case GTU_OPTION: t = (a.f32 > b.f32 ) || isNaN(a.f32) || isNaN(b.f32); break; + case GEU_OPTION: t = (a.f32 >= b.f32) || isNaN(a.f32) || isNaN(b.f32); break; + case NUM_OPTION: t = !isNaN(a.f32) && !isNaN(b.f32); break; + case NAN_OPTION: t = isNaN(a.f32) || isNaN(b.f32); break; + default: + assert(0); + } + break; + case F64_TYPE: + case FF64_TYPE: + switch (cmpop) { + case EQ_OPTION: t = (a.f64 == b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; + case NE_OPTION: t = (a.f64 != b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; + case LT_OPTION: t = (a.f64 < b.f64 ) && !isNaN(a.f64) && !isNaN(b.f64); break; + case LE_OPTION: t = (a.f64 <= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; + case GT_OPTION: t = (a.f64 > b.f64 ) && !isNaN(a.f64) && !isNaN(b.f64); break; + case GE_OPTION: t = (a.f64 >= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; + case EQU_OPTION: t = (a.f64 == b.f64) || isNaN(a.f64) || isNaN(b.f64); break; + case NEU_OPTION: t = (a.f64 != b.f64) || isNaN(a.f64) || isNaN(b.f64); break; + case LTU_OPTION: t = (a.f64 < b.f64 ) || isNaN(a.f64) || isNaN(b.f64); break; + case LEU_OPTION: t = (a.f64 <= b.f64) || isNaN(a.f64) || isNaN(b.f64); break; + case GTU_OPTION: t = (a.f64 > b.f64 ) || isNaN(a.f64) || isNaN(b.f64); break; + case GEU_OPTION: t = (a.f64 >= b.f64) || isNaN(a.f64) || isNaN(b.f64); break; + case NUM_OPTION: t = !isNaN(a.f64) && !isNaN(b.f64); break; + case NAN_OPTION: t = isNaN(a.f64) || isNaN(b.f64); break; + default: + assert(0); + } + break; + default: assert(0); break; + } + + return t; +} + +void setp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b; + + int t=0; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + assert( pI->get_num_operands() < 4 ); // or need to deal with "c" operand / boolOp + + unsigned type = pI->get_type(); + unsigned cmpop = pI->get_cmpop(); + a = thread->get_operand_value(src1, dst, type, thread, 1); + b = thread->get_operand_value(src2, dst, type, thread, 1); + + t = CmpOp(type,a,b,cmpop); + + ptx_reg_t data; + + //the way ptxplus handles the zero flag, 1 = false and 0 = true + data.pred = (t==0); //inverting predicate since ptxplus uses "1" for a set zero flag + + thread->set_operand_value(dst,data, PRED_TYPE, thread, pI); +} + +void set_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b; + + int t=0; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + assert( pI->get_num_operands() < 4 ); // or need to deal with "c" operand / boolOp + + unsigned src_type = pI->get_type2(); + unsigned cmpop = pI->get_cmpop(); + + a = thread->get_operand_value(src1, dst, src_type, thread, 1); + b = thread->get_operand_value(src2, dst, src_type, thread, 1); + + // Take abs of first operand if needed + if(pI->is_abs()) { + switch ( src_type ) { + case S16_TYPE: a.s16 = my_abs(a.s16); break; + case S32_TYPE: a.s32 = my_abs(a.s32); break; + case S64_TYPE: a.s64 = my_abs(a.s64); break; + case U16_TYPE: a.u16 = a.u16; break; + case U32_TYPE: a.u32 = my_abs(a.u32); break; + case U64_TYPE: a.u64 = my_abs(a.u64); break; + case F32_TYPE: a.f32 = my_abs(a.f32); break; + case F64_TYPE: case FF64_TYPE: a.f64 = my_abs(a.f64); break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + } + + t = CmpOp(src_type,a,b,cmpop); + + ptx_reg_t data; + if ( isFloat(pI->get_type()) ) { + data.f32 = (t!=0)?1.0f:0.0f; + } else { + data.u32 = (t!=0)?0xFFFFFFFF:0; + } + + thread->set_operand_value(dst, data, pI->get_type(), thread, pI); + +} + +void shfl_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) +{ + unsigned i_type = pI->get_type(); + int tid = inst.warp_id() * core->get_warp_size(); + ptx_thread_info *thread = core->get_thread_info()[tid]; + ptx_warp_info *warp_info = thread->m_warp_info; + int lane = warp_info->get_done_threads(); + thread = core->get_thread_info()[tid + lane]; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + int bval = (thread->get_operand_value(src2, dst, i_type, thread, 1)).u32; + int cval = (thread->get_operand_value(src3, dst, i_type, thread, 1)).u32; + int mask = cval >> 8; + bval &= 0x1F; + cval &= 0x1F; + + int maxLane = (lane & mask) | (cval & ~mask); + int minLane = lane & mask; + + int src_idx; + unsigned p; + switch(pI->shfl_op()) { + case UP_OPTION: + src_idx = lane - bval; + p = (src_idx >= maxLane); + break; + case DOWN_OPTION: + src_idx = lane + bval; + p = (src_idx <= maxLane); + break; + case BFLY_OPTION: + src_idx = lane ^ bval; + p = (src_idx <= maxLane); + break; + case IDX_OPTION: + src_idx = minLane | (bval & ~mask); + p = (src_idx <= maxLane); + break; + default: + printf("GPGPU-Sim PTX: ERROR: Invalid shfl option\n"); + assert(0); + break; + } + // copy from own lane + if (!p) src_idx = lane; + + // copy input from lane src_idx + ptx_reg_t data; + if (inst.active(src_idx)) { + ptx_thread_info *source = core->get_thread_info()[tid + src_idx]; + data = source->get_operand_value(src1, dst, i_type, source, 1); + } else { + printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for inactive threads in a warp\n"); + data.u32 = 0; + } + thread->set_operand_value(dst, data, i_type, thread, pI); + + /* + TODO: deal with predicates appropriately using the following pseudocode: + if (!isGuardPredicateTrue(src_idx)) { + printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for predicated-off threads in a warp\n"); + } + if (dest predicate selected) data.pred = p; + */ + + // keep track of the number of threads that have executed in the warp + warp_info->inc_done_threads(); + if (warp_info->get_done_threads() == inst.active_count()) { + warp_info->reset_done_threads(); + } +} + +void shl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + + switch ( i_type ) { + case B16_TYPE: + case U16_TYPE: + if ( b.u16 >= 16 ) + d.u16 = 0; + else + d.u16 = (unsigned short) ((a.u16 << b.u16) & 0xFFFF); + break; + case B32_TYPE: + case U32_TYPE: + if ( b.u32 >= 32 ) + d.u32 = 0; + else + d.u32 = (unsigned) ((a.u32 << b.u32) & 0xFFFFFFFF); + break; + case B64_TYPE: + case U64_TYPE: + if ( b.u32 >= 64 ) + d.u64 = 0; + else + d.u64 = (a.u64 << b.u64); + break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +void shr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + + + switch ( i_type ) { + case U16_TYPE: + case B16_TYPE: + if ( b.u16 < 16 ) + d.u16 = (unsigned short) ((a.u16 >> b.u16) & 0xFFFF); + else + d.u16 = 0; + break; + case U32_TYPE: + case B32_TYPE: + if ( b.u32 < 32 ) + d.u32 = (unsigned) ((a.u32 >> b.u32) & 0xFFFFFFFF); + else + d.u32 = 0; + break; + case U64_TYPE: + case B64_TYPE: + if ( b.u32 < 64 ) + d.u64 = (a.u64 >> b.u64); + else + d.u64 = 0; + break; + case S16_TYPE: + if ( b.u16 < 16 ) + d.s64 = (a.s16 >> b.s16); + else { + if ( a.s16 < 0 ) { + d.s64 = -1; + } else { + d.s64 = 0; + } + } + break; + case S32_TYPE: + if ( b.u32 < 32 ) + d.s64 = (a.s32 >> b.s32); + else { + if ( a.s32 < 0 ) { + d.s64 = -1; + } else { + d.s64 = 0; + } + } + break; + case S64_TYPE: + if ( b.u64 < 64 ) + d.s64 = (a.s64 >> b.u64); + else { + if ( a.s64 < 0 ) { + if ( b.s32 < 0 ) { + d.u64 = -1; + d.s32 = 0; + } else { + d.s64 = -1; + } + } else { + d.s64 = 0; + } + } + break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +void sin_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + + switch ( i_type ) { + case F32_TYPE: + d.f32 = sin(a.f32); + break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +void slct_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + ptx_reg_t a, b, c, d; + + unsigned i_type = pI->get_type(); + unsigned c_type = pI->get_type2(); + bool t = false; + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, c_type, thread, 1); + + switch ( c_type ) { + case S32_TYPE: t = c.s32 >= 0; break; + case F32_TYPE: t = c.f32 >= 0; break; + default: assert(0); + } + + switch ( i_type ) { + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: d.u16 = t?a.u16:b.u16; break; + case F32_TYPE: + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: d.u32 = t?a.u32:b.u32; break; + case F64_TYPE: + case FF64_TYPE: + case B64_TYPE: + case S64_TYPE: + case U64_TYPE: d.u64 = t?a.u64:b.u64; break; + default: assert(0); + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +void sqrt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + + switch ( i_type ) { + case F32_TYPE: + if ( a.f32 < 0 ) + d.f32 = nanf(""); + else + d.f32 = sqrt(a.f32); break; + case F64_TYPE: + case FF64_TYPE: + if ( a.f64 < 0 ) + d.f64 = nan(""); + else + d.f64 = sqrt(a.f64); break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst,d, i_type, thread, pI); +} + +void ssy_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + //printf("Execution Warning: unimplemented ssy instruction is treated as a nop\n"); + // TODO: add implementation +} + +void st_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); //may be scalar or vector of regs + unsigned type = pI->get_type(); + ptx_reg_t addr_reg = thread->get_operand_value(dst, dst, type, thread, 1); + ptx_reg_t data; + memory_space_t space = pI->get_space(); + unsigned vector_spec = pI->get_vector(); + + memory_space *mem = NULL; + addr_t addr = addr_reg.u32; + + decode_space(space,thread,dst,mem,addr); + + size_t size; + int t; + type_info_key::type_decode(type,size,t); + + if (!vector_spec) { + data = thread->get_operand_value(src1, dst, type, thread, 1); + mem->write(addr,size/8,&data.s64,thread,pI); + } else { + if (vector_spec == V2_TYPE) { + ptx_reg_t* ptx_regs = new ptx_reg_t[2]; + thread->get_vector_operand_values(src1, ptx_regs, 2); + mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); + mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); + delete [] ptx_regs; + } + if (vector_spec == V3_TYPE) { + ptx_reg_t* ptx_regs = new ptx_reg_t[3]; + thread->get_vector_operand_values(src1, ptx_regs, 3); + mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); + mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); + mem->write(addr+2*size/8,size/8,&ptx_regs[2].s64,thread,pI); + delete [] ptx_regs; + } + if (vector_spec == V4_TYPE) { + ptx_reg_t* ptx_regs = new ptx_reg_t[4]; + thread->get_vector_operand_values(src1, ptx_regs, 4); + mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); + mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); + mem->write(addr+2*size/8,size/8,&ptx_regs[2].s64,thread,pI); + mem->write(addr+3*size/8,size/8,&ptx_regs[3].s64,thread,pI); + delete [] ptx_regs; + } + } + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; +} + +void sub_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t data; + int overflow = 0; + int carry = 0; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + //performs addition. Sets carry and overflow if needed. + //the constant is added in during subtraction so the carry bit is set properly. + switch ( i_type ) { + case S8_TYPE: + data.s64 = (src1_data.s64 & 0xFF) - (src2_data.s64 & 0xFF) + 0x100; + if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) != 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } + carry = (data.s32 & 0x100)>>8; + break; + case S16_TYPE: + data.s64 = (src1_data.s64 & 0xFFFF) - (src2_data.s64 & 0xFFFF) + 0x10000; + if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) != 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } + carry = (data.s32 & 0x10000)>>16; + break; + case S32_TYPE: + data.s64 = (src1_data.s64 & 0xFFFFFFFF) - (src2_data.s64 & 0xFFFFFFFF) + 0x100000000; + if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) != 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } + carry = ((data.u64)>>32) & 0x0001; + break; + case S64_TYPE: + data.s64 = src1_data.s64 - src2_data.s64; break; + case B8_TYPE: + case U8_TYPE: + data.u64 = (src1_data.u64 & 0xFF) - (src2_data.u64 & 0xFF) + 0x100; + carry = (data.u64 & 0x100)>>8; + break; + case B16_TYPE: + case U16_TYPE: + data.u64 = (src1_data.u64 & 0xFFFF) - (src2_data.u64 & 0xFFFF) + 0x10000; + carry = (data.u64 & 0x10000)>>16; + break; + case B32_TYPE: + case U32_TYPE: + data.u64 = (src1_data.u64 & 0xFFFFFFFF) - (src2_data.u64 & 0xFFFFFFFF) + 0x100000000; + carry = (data.u64 & 0x100000000)>>32; + break; + case B64_TYPE: + case U64_TYPE: + data.u64 = src1_data.u64 - src2_data.u64; break; + case F16_TYPE: assert(0); break; + case F32_TYPE: data.f32 = src1_data.f32 - src2_data.f32; break; + case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 - src2_data.f64; break; + default: assert(0); break; + } + + thread->set_operand_value(dst,data, i_type, thread, pI, overflow, carry); +} + +void nop_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + // Do nothing +} + +void subc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void suld_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void sured_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void sust_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void suq_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } + +ptx_reg_t* ptx_tex_regs = NULL; + +union intfloat { + int a; + float b; +}; + +float reduce_precision( float x, unsigned bits ) +{ + intfloat tmp; + tmp.b = x; + int v = tmp.a; + int man = v & ((1<<23)-1); + int mask = ((1<= mx) nx -= elem_size; + unsigned ny = (y >= my)? my - 1 : y; + return nx + mx*ny; +} + +typedef unsigned (*texAddr_t) (unsigned x, unsigned y, unsigned mx, unsigned my, size_t elem_size); +float tex_linf_sampling(memory_space* mem, unsigned tex_array_base, + int x, int y, unsigned int width, unsigned int height, size_t elem_size, + float alpha, float beta, texAddr_t b_lim) +{ + float Tij; + float Ti1j; + float Tij1; + float Ti1j1; + + mem->read(tex_array_base + b_lim(x,y,width,height,elem_size), 4, &Tij); + mem->read(tex_array_base + b_lim(x+elem_size,y,width,height,elem_size), 4, &Ti1j); + mem->read(tex_array_base + b_lim(x,y+1,width,height,elem_size), 4, &Tij1); + mem->read(tex_array_base + b_lim(x+elem_size,y+1,width,height,elem_size), 4, &Ti1j1); + + float sample = (1-alpha)*(1-beta)*Tij + + alpha*(1-beta)*Ti1j + + (1-alpha)*beta*Tij1 + + alpha*beta*Ti1j1; + + return sample; +} + +float textureNormalizeElementSigned(int element, int bits) +{ + if (bits) { + int maxN = (1 << bits) - 1; + // removing upper bits + element &= maxN; + // normalizing the number to [-1.0,1.0] + maxN >>= 1; + float output = (float) element / maxN; + if (output < -1.0f) output = -1.0f; + return output; + } else { + return 0.0f; + } +} + +float textureNormalizeElementUnsigned(unsigned int element, int bits) +{ + if (bits) { + unsigned int maxN = (1 << bits) - 1; + // removing upper bits and normalizing the number to [0.0,1.0] + return (float)(element & maxN) / maxN; + } else { + return 0.0f; + } +} + +void textureNormalizeOutput( const struct cudaChannelFormatDesc& desc, ptx_reg_t& datax, ptx_reg_t& datay, ptx_reg_t& dataz, ptx_reg_t& dataw ) +{ + if (desc.f == cudaChannelFormatKindSigned) { + datax.f32 = textureNormalizeElementSigned( datax.s32, desc.x ); + datay.f32 = textureNormalizeElementSigned( datay.s32, desc.y ); + dataz.f32 = textureNormalizeElementSigned( dataz.s32, desc.z ); + dataw.f32 = textureNormalizeElementSigned( dataw.s32, desc.w ); + } else if (desc.f == cudaChannelFormatKindUnsigned) { + datax.f32 = textureNormalizeElementUnsigned( datax.u32, desc.x ); + datay.f32 = textureNormalizeElementUnsigned( datay.u32, desc.y ); + dataz.f32 = textureNormalizeElementUnsigned( dataz.u32, desc.z ); + dataw.f32 = textureNormalizeElementUnsigned( dataw.u32, desc.w ); + } else { + assert(0 && "Undefined texture read mode: cudaReadModeNormalizedFloat expect integer elements"); + } +} + +void tex_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + unsigned dimension = pI->dimension(); + const operand_info &dst = pI->dst(); //the registers to which fetched texel will be placed + const operand_info &src1 = pI->src1(); //the name of the texture + const operand_info &src2 = pI->src2(); //the vector registers containing coordinates of the texel to be fetched + + std::string texname = src1.name(); + unsigned to_type = pI->get_type(); + unsigned c_type = pI->get_type2(); + fflush(stdout); + ptx_reg_t data1, data2, data3, data4; + if (!ptx_tex_regs) ptx_tex_regs = new ptx_reg_t[4]; + unsigned nelem = src2.get_vect_nelem(); + thread->get_vector_operand_values(src2, ptx_tex_regs, nelem); //ptx_reg should be 4 entry vector type...coordinates into texture + + gpgpu_t *gpu = thread->get_gpu(); + const struct textureReference* texref = gpu->get_texref(texname); + const struct cudaArray* cuArray = gpu->get_texarray(texref); + const struct textureInfo* texInfo = gpu->get_texinfo(texref); + const struct textureReferenceAttr* texAttr = gpu->get_texattr(texref); + + //assume always 2D f32 input + //access array with src2 coordinates + memory_space *mem = thread->get_global_memory(); + float x_f32, y_f32; + size_t size; + int t; + unsigned tex_array_base; + unsigned int width = 0, height = 0; + int x = 0; + int y = 0; + unsigned tex_array_index; + float alpha=0, beta=0; + + type_info_key::type_decode(to_type,size,t); + tex_array_base = cuArray->devPtr32; + + switch (dimension) { + case GEOM_MODIFIER_1D: + width = cuArray->width; + height = cuArray->height; + if (texref->normalized) { + assert(c_type == F32_TYPE); + x_f32 = ptx_tex_regs[0].f32; + if (texref->addressMode[0] == cudaAddressModeClamp) { + x_f32 = (x_f32 > 1.0)? 1.0 : x_f32; + x_f32 = (x_f32 < 0.0)? 0.0 : x_f32; + } else if (texref->addressMode[0] == cudaAddressModeWrap) { + x_f32 = x_f32 - floor(x_f32); + } + + if( texref->filterMode == cudaFilterModeLinear ) { + float xb = x_f32 * width - 0.5; + alpha = xb - floor(xb); + alpha = reduce_precision(alpha,9); + beta = 0.0; + + x = (int)floor(xb); + y = 0; + } else { + x = (int) floor(x_f32 * width); + y = 0; + } + } else { + switch ( c_type ) { + case S32_TYPE: + x = ptx_tex_regs[0].s32; + assert(texref->filterMode == cudaFilterModePoint); + break; + case F32_TYPE: + x_f32 = ptx_tex_regs[0].f32; + alpha = x_f32 - floor(x_f32); // offset into subtexel (for linear sampling) + x = (int) x_f32; + break; + default: assert(0 && "Unsupported texture coordinate type."); + } + // handle texture fetch that exceeded boundaries + if (texref->addressMode[0] == cudaAddressModeClamp) { + x = (x > width - 1)? (width - 1) : x; + x = (x < 0)? 0 : x; + } else if (texref->addressMode[0] == cudaAddressModeWrap) { + x = x % width; + } + } + width *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; + x *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; + tex_array_index = tex_array_base + x; + + break; + case GEOM_MODIFIER_2D: + width = cuArray->width; + height = cuArray->height; + if (texref->normalized) { + x_f32 = reduce_precision(ptx_tex_regs[0].f32,16); + y_f32 = reduce_precision(ptx_tex_regs[1].f32,15); + + if (texref->addressMode[0]) {//clamp + if (x_f32<0) x_f32 = 0; + if (x_f32>=1) x_f32 = 1 - 1/x_f32; + } else {//wrap + x_f32 = x_f32 - floor(x_f32); + } + if (texref->addressMode[1]) {//clamp + if (y_f32<0) y_f32 = 0; + if (y_f32>=1) y_f32 = 1 - 1/y_f32; + } else {//wrap + y_f32 = y_f32 - floor(y_f32); + } + + if( texref->filterMode == cudaFilterModeLinear ) { + float xb = x_f32 * width - 0.5; + float yb = y_f32 * height - 0.5; + alpha = xb - floor(xb); + beta = yb - floor(yb); + alpha = reduce_precision(alpha,9); + beta = reduce_precision(beta,9); + + x = (int)floor(xb); + y = (int)floor(yb); + } else { + x = (int) floor(x_f32 * width); + y = (int) floor(y_f32 * height); + } + } else { + x_f32 = ptx_tex_regs[0].f32; + y_f32 = ptx_tex_regs[1].f32; + + alpha = x_f32 - floor(x_f32); + beta = y_f32 - floor(y_f32); + + x = (int) x_f32; + y = (int) y_f32; + if (texref->addressMode[0]) {//clamp + if (x<0) x = 0; + if (x>= (int)width) x = width-1; + } else {//wrap + x = x % width; + if (x < 0) x*= -1; + } + if (texref->addressMode[1]) {//clamp + if (y<0) y = 0; + if (y>= (int)height) y = height -1; + } else {//wrap + y = y % height; + if (y < 0) y *= -1; + } + } + + width *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; + x *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; + tex_array_index = tex_array_base + (x + width*y); + break; + default: + assert(0); break; + } + switch ( to_type ) { + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case B8_TYPE: + case B16_TYPE: + case B32_TYPE: + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: { + unsigned long long elementOffset = 0; // offset into the next element + mem->read( tex_array_index, cuArray->desc.x/8, &data1.u32); + elementOffset += cuArray->desc.x/8; + if (cuArray->desc.y) { + mem->read( tex_array_index + elementOffset, cuArray->desc.y/8, &data2.u32); + elementOffset += cuArray->desc.y/8; + if (cuArray->desc.z) { + mem->read( tex_array_index + elementOffset, cuArray->desc.z/8, &data3.u32); + elementOffset += cuArray->desc.z/8; + if (cuArray->desc.w) + mem->read( tex_array_index + elementOffset, cuArray->desc.w/8, &data4.u32); + } + } + break; + } + case B64_TYPE: + case U64_TYPE: + case S64_TYPE: + mem->read( tex_array_index, 8, &data1.u64); + if (cuArray->desc.y) { + mem->read( tex_array_index+8, 8, &data2.u64); + if (cuArray->desc.z) { + mem->read( tex_array_index+16, 8, &data3.u64); + if (cuArray->desc.w) + mem->read( tex_array_index+24, 8, &data4.u64); + } + } + break; + case F16_TYPE: assert(0); break; + case F32_TYPE: { + if( texref->filterMode == cudaFilterModeLinear ) { + texAddr_t b_lim = wrap; + if ( texref->addressMode[0] == cudaAddressModeClamp ) { + b_lim = clamp; + } + size_t elem_size = (cuArray->desc.x + cuArray->desc.y + cuArray->desc.z + cuArray->desc.w) / 8; + size_t elem_ofst = 0; + + data1.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.x / 8; + if (cuArray->desc.y) { + data2.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.y / 8; + if (cuArray->desc.z) { + data3.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.z / 8; + if (cuArray->desc.w) + data4.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); + } + } + } else { + mem->read( tex_array_index, cuArray->desc.x/8, &data1.f32); + if (cuArray->desc.y) { + mem->read( tex_array_index+4, cuArray->desc.y/8, &data2.f32); + if (cuArray->desc.z) { + mem->read( tex_array_index+8, cuArray->desc.z/8, &data3.f32); + if (cuArray->desc.w) + mem->read( tex_array_index+12, cuArray->desc.w/8, &data4.f32); + } + } + } + } break; + case F64_TYPE: + case FF64_TYPE: + mem->read( tex_array_index, 8, &data1.f64); + if (cuArray->desc.y) { + mem->read( tex_array_index+8, 8, &data2.f64); + if (cuArray->desc.z) { + mem->read( tex_array_index+16, 8, &data3.f64); + if (cuArray->desc.w) + mem->read( tex_array_index+24, 8, &data4.f64); + } + } + break; + default: assert(0); break; + } + int x_block_coord, y_block_coord, memreqindex, blockoffset; + + switch (dimension) { + case GEOM_MODIFIER_1D: + thread->m_last_effective_address = tex_array_index; + break; + case GEOM_MODIFIER_2D: + x_block_coord = x >> (texInfo->Tx_numbits + texInfo->texel_size_numbits); + y_block_coord = y >> texInfo->Ty_numbits; + + memreqindex = ((y_block_coord*cuArray->width/texInfo->Tx)+x_block_coord)<<6; + + blockoffset = (x%(texInfo->Tx*texInfo->texel_size) + (y%(texInfo->Ty)<<(texInfo->Tx_numbits + texInfo->texel_size_numbits))); + memreqindex += blockoffset; + thread->m_last_effective_address = tex_array_base + memreqindex;//tex_array_index; + break; + default: + assert(0); + } + thread->m_last_memory_space = tex_space; + + // normalize output into floating point numbers according to the texture read mode + if (texAttr->m_readmode == cudaReadModeNormalizedFloat) { + textureNormalizeOutput(cuArray->desc, data1, data2, data3, data4); + } else { + assert(texAttr->m_readmode == cudaReadModeElementType); + } + + thread->set_vector_operand_values(dst,data1,data2,data3,data4); +} + +void txq_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void trap_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vabsdiff_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vadd_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vmad_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vmax_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vmin_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vset_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vshl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vshr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void vsub_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } + +void vote_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + static bool first_in_warp = true; + static bool and_all; + static bool or_all; + static unsigned int ballot_result; + static std::list threads_in_warp; + static unsigned last_tid; + + if( first_in_warp ) { + first_in_warp = false; + threads_in_warp.clear(); + and_all = true; + or_all = false; + ballot_result = 0; + int offset=31; + while( (offset>=0) && !pI->active(offset) ) + offset--; + assert( offset >= 0 ); + last_tid = (thread->get_hw_tid() - (thread->get_hw_tid()%pI->warp_size())) + offset; + } + + ptx_reg_t src1_data; + const operand_info &src1 = pI->src1(); + src1_data = thread->get_operand_value(src1, pI->dst(), PRED_TYPE, thread, 1); + + //predicate value was changed so the lowest bit being set means the zero flag is set. + //As a result, the value of src1_data.pred must be inverted to get proper behavior + bool pred_value = !(src1_data.pred & 0x0001); + bool invert = src1.is_neg_pred(); + + threads_in_warp.push_back(thread); + and_all &= (invert ^ pred_value); + or_all |= (invert ^ pred_value); + + // vote.ballot + if (invert ^ pred_value) { + int lane_id = thread->get_hw_tid() % pI->warp_size(); + ballot_result |= (1 << lane_id); + } + + if( thread->get_hw_tid() == last_tid ) { + if (pI->vote_mode() == ptx_instruction::vote_ballot) { + ptx_reg_t data = ballot_result; + for( std::list::iterator t=threads_in_warp.begin(); t!=threads_in_warp.end(); ++t ) { + const operand_info &dst = pI->dst(); + (*t)->set_operand_value(dst,data, pI->get_type(), (*t), pI); + } + } else { + bool pred_value = false; + + switch( pI->vote_mode() ) { + case ptx_instruction::vote_any: pred_value = or_all; break; + case ptx_instruction::vote_all: pred_value = and_all; break; + case ptx_instruction::vote_uni: pred_value = (or_all ^ and_all); break; + default: + abort(); + } + ptx_reg_t data; + data.pred = pred_value?0:1; //the way ptxplus handles the zero flag, 1 = false and 0 = true + + for( std::list::iterator t=threads_in_warp.begin(); t!=threads_in_warp.end(); ++t ) { + const operand_info &dst = pI->dst(); + (*t)->set_operand_value(dst,data, PRED_TYPE, (*t), pI); + } + } + first_in_warp = true; + } +} + +void xor_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + //the way ptxplus handles predicates: 1 = false and 0 = true + if(i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) ^ ~(src2_data.pred)); + else + data.u64 = src1_data.u64 ^ src2_data.u64; + + thread->set_operand_value(dst,data, i_type, thread, pI); +} + +void inst_not_implemented( const ptx_instruction * pI ) +{ + printf("GPGPU-Sim PTX: ERROR (%s:%u) instruction \"%s\" not (yet) implemented\n", + pI->source_file(), + pI->source_line(), + pI->get_opcode_cstr() ); + abort(); +} + +ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread) +{ + ptx_reg_t result; + memory_space *mem = NULL; + size_t size; + int t; + result.u64=0; + + //complete other cases for reading from memory, such as reading from other const memory + if(opInfo.get_addr_space() == global_space) + { + mem = thread->get_global_memory(); + type_info_key::type_decode(type,size,t); + mem->read(opData.u32,size/8,&result.u64); + if( type == S16_TYPE || type == S32_TYPE ) + sign_extend(result,size,dstInfo); + } + else if(opInfo.get_addr_space() == shared_space) + { + mem = thread->m_shared_mem; + type_info_key::type_decode(type,size,t); + mem->read(opData.u32,size/8,&result.u64); + + if( type == S16_TYPE || type == S32_TYPE ) + sign_extend(result,size,dstInfo); + + } + else if(opInfo.get_addr_space() == const_space) + { + mem = thread->get_global_memory(); + type_info_key::type_decode(type,size,t); + + mem->read((opData.u32 + opInfo.get_const_mem_offset()),size/8,&result.u64); + + if( type == S16_TYPE || type == S32_TYPE ) + sign_extend(result,size,dstInfo); + } + else + { + result = opData; + } + + if(opInfo.get_operand_lohi() == 1) + { + result.u64 = result.u64 & 0xFFFF; + } + else if(opInfo.get_operand_lohi() == 2) + { + result.u64 = (result.u64>>16) & 0xFFFF; + } + + if(opInfo.get_operand_neg() == true) { + result.f32 = -result.f32; + } + + return result; +} + diff --git a/src/cuda-sim/ptx_loader.cc~ b/src/cuda-sim/ptx_loader.cc~ new file mode 100644 index 0000000..c922b18 --- /dev/null +++ b/src/cuda-sim/ptx_loader.cc~ @@ -0,0 +1,462 @@ +// Copyright (c) 2009-2011, Tor M. Aamodt +// The University of British Columbia +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// Redistributions in binary form must reproduce the above copyright notice, this +// list of conditions and the following disclaimer in the documentation and/or +// other materials provided with the distribution. +// Neither the name of The University of British Columbia nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "ptx_loader.h" +#include "ptx_ir.h" +#include "cuda-sim.h" +#include "ptx_parser.h" +#include +#include +#include +#include + +/// globals + +memory_space *g_global_mem; +memory_space *g_tex_mem; +memory_space *g_surf_mem; +memory_space *g_param_mem; +bool g_override_embedded_ptx = false; + +/// extern prototypes + +extern int ptx_parse(); +extern int ptx__scan_string(const char*); + +extern std::map get_duplicate(); + +const char *g_ptxinfo_filename; +extern int ptxinfo_parse(); +extern int ptxinfo_debug; +extern FILE *ptxinfo_in; + +static bool g_save_embedded_ptx; +bool g_keep_intermediate_files; +bool m_ptx_save_converted_ptxplus; + +bool keep_intermediate_files() {return g_keep_intermediate_files;} + +void ptx_reg_options(option_parser_t opp) +{ + option_parser_register(opp, "-save_embedded_ptx", OPT_BOOL, &g_save_embedded_ptx, + "saves ptx files embedded in binary as .ptx", + "0"); + option_parser_register(opp, "-keep", OPT_BOOL, &g_keep_intermediate_files, + "keep intermediate files created by GPGPU-Sim when interfacing with external programs", + "0"); + option_parser_register(opp, "-gpgpu_ptx_save_converted_ptxplus", OPT_BOOL, + &m_ptx_save_converted_ptxplus, + "Saved converted ptxplus to a file", + "0"); +} + +void print_ptx_file( const char *p, unsigned source_num, const char *filename ) +{ + printf("\nGPGPU-Sim PTX: file _%u.ptx contents:\n\n", source_num ); + char *s = strdup(p); + char *t = s; + unsigned n=1; + while ( *t != '\0' ) { + char *u = t; + while ( (*u != '\n') && (*u != '\0') ) u++; + unsigned last = (*u == '\0'); + *u = '\0'; + const ptx_instruction *pI = ptx_instruction_lookup(filename,n); + char pc[64]; + if( pI && pI->get_PC() ) + snprintf(pc,64,"%4u", pI->get_PC() ); + else + snprintf(pc,64," "); + printf(" _%u.ptx %4u (pc=%s): %s\n", source_num, n, pc, t ); + if ( last ) break; + t = u+1; + n++; + } + free(s); + fflush(stdout); +} + +char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptxfilename, const std::string elffilename, const std::string sassfilename) +{ + + printf("GPGPU-Sim PTX: converting EMBEDDED .ptx file to ptxplus \n"); + + char fname_ptxplus[1024]; + snprintf(fname_ptxplus,1024,"_ptxplus_XXXXXX"); + int fd4=mkstemp(fname_ptxplus); + close(fd4); + + // Run cuobjdump_to_ptxplus + char commandline[1024]; + int result; + snprintf(commandline, 1024, "$GPGPUSIM_ROOT/build/$GPGPUSIM_CONFIG/cuobjdump_to_ptxplus/cuobjdump_to_ptxplus %s %s %s %s", + ptxfilename.c_str(), + sassfilename.c_str(), + elffilename.c_str(), + fname_ptxplus); + fflush(stdout); + printf("GPGPU-Sim PTX: calling cuobjdump_to_ptxplus\ncommandline: %s\n", commandline); + result = system(commandline); + if(result){printf("GPGPU-Sim PTX: ERROR ** could not execute %s\n", commandline); exit(1);} + + + // Get ptxplus from file + std::ifstream fileStream(fname_ptxplus, std::ios::in); + std::string text, line; + while(getline(fileStream,line)) { + text += (line + "\n"); + } + fileStream.close(); + + char* ptxplus_str = new char [strlen(text.c_str())+1]; + strcpy(ptxplus_str, text.c_str()); + + if (!m_ptx_save_converted_ptxplus){ + char rm_commandline[1024]; + + snprintf(rm_commandline,1024,"rm -f %s", fname_ptxplus); + + printf("GPGPU-Sim PTX: removing temporary files using \"%s\"\n", rm_commandline); + int rm_result = system(rm_commandline); + if( rm_result != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", rm_result); + exit(1); + } + } + printf("GPGPU-Sim PTX: DONE converting EMBEDDED .ptx file to ptxplus \n"); + + return ptxplus_str; +} + + +symbol_table *gpgpu_ptx_sim_load_ptx_from_string( const char *p, unsigned source_num ) +{ + char buf[1024]; + snprintf(buf,1024,"_%u.ptx", source_num ); + if( g_save_embedded_ptx ) { + FILE *fp = fopen(buf,"w"); + fprintf(fp,"%s",p); + fclose(fp); + } + symbol_table *symtab=init_parser(buf); + ptx__scan_string(p); + int errors = ptx_parse (); + if ( errors ) { + char fname[1024]; + snprintf(fname,1024,"_ptx_errors_XXXXXX"); + int fd=mkstemp(fname); + close(fd); + printf("GPGPU-Sim PTX: parser error detected, exiting... but first extracting .ptx to \"%s\"\n", fname); + FILE *ptxfile = fopen(fname,"w"); + fprintf(ptxfile,"%s", p ); + fclose(ptxfile); + abort(); + exit(40); + } + + //if ( g_debug_execution >= 100 ) + print_ptx_file(p,source_num,buf); + + printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n",buf); + return symtab; +} + +void fix_duplicate_errors(char fname2[1024]) { + char tempfile[1024] = "_temp_ptx"; + char commandline[1024]; + + // change the name of the ptx file to _temp_ptx + snprintf(commandline,1024,"mv %s %s",fname2,tempfile); + printf("Running: %s\n", commandline); + int result = system(commandline); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", fname2, tempfile); + exit(1); + } + + // store all of the ptx into a char array + FILE *ptxsource = fopen(tempfile,"r"); + fseek(ptxsource, 0, SEEK_END); + long filesize = ftell(ptxsource); + rewind(ptxsource); + char *ptxdata = (char*)malloc((filesize+1)*sizeof(char)); + fread(ptxdata, filesize, 1, ptxsource); + fclose(ptxsource); + + FILE *ptxdest = fopen(fname2,"w"); + std::map duplicate = get_duplicate(); + unsigned offset; + unsigned oldlinenum = 1; + unsigned linenum; + char *startptr = ptxdata; + char *funcptr; + char *tempptr = ptxdata - 1; + char *lineptr = ptxdata - 1; + + // recreate the ptx file without duplications + for ( std::map::iterator iter = duplicate.begin(); + iter != duplicate.end(); + iter++){ + // find the line of the next error + linenum = iter->first; + for (int i = oldlinenum; i < linenum; i++) { + lineptr = strchr(lineptr + 1, '\n'); + } + + // find the end of the current section to be copied over + // then find the start of the next section that will be copied + if (strcmp("function", iter->second) == 0) { + // get location of most recent .func + while (tempptr < lineptr && tempptr != NULL) { + funcptr = tempptr; + tempptr = strstr(funcptr + 1, ".func"); + } + + // get the start of the previous line + offset = 0; + while (*(funcptr - offset) != '\n') offset++; + + fwrite(startptr, sizeof(char), funcptr - offset + 1 - startptr, ptxdest); + + //find next location of startptr + if (*(lineptr + 3) == ';') { + // for function definitions + startptr = lineptr + 5; + } else if (*(lineptr + 3) == '{') { + // for functions enclosed with curly brackets + offset = 5; + unsigned bracket = 1; + while (bracket != 0) { + if (*(lineptr + offset) == '{') bracket++; + else if (*(lineptr + offset) == '}') bracket--; + offset++; + } + startptr = lineptr + offset + 1; + } else { + printf("GPGPU-Sim PTX: ERROR ** Unrecognized function format\n"); + abort(); + } + } else if (strcmp("variable", iter->second) == 0) { + fwrite(startptr, sizeof(char), (int)(lineptr + 1 - startptr), ptxdest); + + //find next location of startptr + offset = 1; + while (*(lineptr + offset) != '\n') offset++; + startptr = lineptr + offset + 1; + } else { + printf("GPGPU-Sim PTX: ERROR ** Unsupported duplicate type: %s\n", iter->second); + } + + oldlinenum = linenum; + } + // copy over the rest of the file + fwrite(startptr, sizeof(char), ptxdata + filesize - startptr, ptxdest); + + // cleanup + free(ptxdata); + fclose(ptxdest); + snprintf(commandline,1024,"rm -f %s",tempfile); + printf("Running: %s\n", commandline); + result = system(commandline); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile); + exit(1); + } +} + +//we need the application name here too. +char* get_app_binary_name(){ + char exe_path[1025]; + char *self_exe_path; +#ifdef __APPLE__ + //AMRUTH: get apple device and check the result. + printf("WARNING: not tested for Apple-mac devices \n"); + abort(); +#else + std::stringstream exec_link; + exec_link << "/proc/self/exe"; + ssize_t path_length = readlink(exec_link.str().c_str(), exe_path, 1024); + assert(path_length != -1); + exe_path[path_length] = '\0'; + + char *token = strtok(exe_path, "/"); + while(token !=NULL){ + self_exe_path = token; + token = strtok(NULL,"/"); + } +#endif + self_exe_path = strtok(self_exe_path, "."); + printf("self exe links to: %s\n", self_exe_path); + return self_exe_path; +} + +void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num, unsigned sm_version ) +{ + //do ptxas for individual files instead of one big embedded ptx. This prevents the duplicate defs and declarations. + char ptx_file[1000]; + char *name=get_app_binary_name(); + char commandline[4096], fname[1024], fname2[1024], final_tempfile_ptxinfo[1024], tempfile_ptxinfo[1024]; + for (int index=1; index <= no_of_ptx; index++){ + snprintf(ptx_file, 1000, "%s.%d.sm_%u.ptx", name, index, sm_version); + snprintf(fname,1024,"_ptx_XXXXXX"); + int fd=mkstemp(fname); + close(fd); + + printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname); + snprintf(commandline,4096,"cat %s > %s",ptx_file, fname); + if (system(commandline) !=0) { + printf("ERROR: %s command failed\n", commandline); + exit(0); + } + + snprintf(fname2,1024,"_ptx2_XXXXXX"); + fd=mkstemp(fname2); + close(fd); + char commandline2[4096]; + snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2); + printf("Running: %s\n", commandline2); + int result = system(commandline2); + if( result != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); + printf(" Ensure you have write access to simulation directory\n"); + printf(" and have \'cat\' and \'sed\' in your path.\n"); + exit(1); + } + + snprintf(tempfile_ptxinfo,1024,"%sinfo",fname); + char extra_flags[1024]; + extra_flags[0]=0; + + #if CUDART_VERSION >= 3000 + if (sm_version == 0) sm_version = 20; + extern bool g_cdp_enabled; + if(!g_cdp_enabled) + snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version); + else + snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version); + #endif + + snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); + result = system(commandline); + if( result != 0 ) { + // 65280 = duplicate errors + if (result == 65280) { + ptxinfo_in = fopen(tempfile_ptxinfo,"r"); + g_ptxinfo_filename = tempfile_ptxinfo; + ptxinfo_parse(); + + fix_duplicate_errors(fname2); + snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: regenerating ptxinfo using \"%s\"\n", commandline); + result = system(commandline); + } + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } + } + } + + //TODO: duplicate code! move it into a function so that it can be reused! + if(no_of_ptx==0) { + //For CDP, we dump everything. So no_of_ptx will be 0. + snprintf(fname,1024,"_ptx_XXXXXX"); + int fd=mkstemp(fname); + close(fd); + + printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname); + FILE *ptxfile = fopen(fname,"w"); + fprintf(ptxfile,"%s", p_for_info); + fclose(ptxfile); + + snprintf(fname2,1024,"_ptx2_XXXXXX"); + fd=mkstemp(fname2); + close(fd); + char commandline2[4096]; + snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2); + printf("Running: %s\n", commandline2); + int result = system(commandline2); + if( result != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); + printf(" Ensure you have write access to simulation directory\n"); + printf(" and have \'cat\' and \'sed\' in your path.\n"); + exit(1); + } + //char tempfile_ptxinfo[1024]; + snprintf(tempfile_ptxinfo,1024,"%sinfo",fname); + char extra_flags[1024]; + extra_flags[0]=0; +#if CUDART_VERSION >= 3000 + snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version); +#endif + + snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); + result = system(commandline); + if( result != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } + } + + //Now that we got resource usage per kernel in a ptx file, we dump all into one file and pass it to rest of the code as usual. + if(no_of_ptx>0){ + char commandline3[4096]; + snprintf(final_tempfile_ptxinfo,1024,"f_tempfile_ptx"); + snprintf(commandline3,4096, "cat *info > %s", final_tempfile_ptxinfo); + if (system(commandline3)!=0) { + printf("ERROR: Either we dont have info files or cat is not working \n"); + printf("ERROR: %s command failed\n",commandline3); + exit(1); + } + } + + ptxinfo_in = fopen(final_tempfile_ptxinfo,"r"); + if(no_of_ptx>0) + g_ptxinfo_filename = final_tempfile_ptxinfo; + else + g_ptxinfo_filename = tempfile_ptxinfo; + ptxinfo_parse(); + + if( ! g_save_embedded_ptx ) { + if(no_of_ptx>0) + snprintf(commandline,1024,"rm -f %s %s %s *info", fname, fname2, final_tempfile_ptxinfo); + else + snprintf(commandline,1024,"rm -f %s %s %s *info", fname, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: removing ptxinfo using \"%s\"\n", commandline); + if( system(commandline) != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while removing temporary files\n"); + exit(1); + } + } +} -- cgit v1.3 From 60017ca1ddbe844a93f631fe2b86bc4101850037 Mon Sep 17 00:00:00 2001 From: Amruth Date: Thu, 19 Apr 2018 18:13:44 -0700 Subject: Crash when array pointers are passed --- README | 27 ++++++++++++++++++++++++++- src/cuda-sim/instructions.cc | 4 +++- src/cuda-sim/ptx_ir.h | 13 +++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) (limited to 'src/cuda-sim/instructions.cc') diff --git a/README b/README index 543177c..bf5aa62 100644 --- a/README +++ b/README @@ -235,6 +235,14 @@ The documentation resides at doc/doxygen/html. Step 3: Run ============ +Before we run, we need to make sure the application's executable file is dynamically linked to CUDA runtime library. This can be done during compilation of your program by introducing the nvcc flag "--cudart shared" in makefile (quotes should be excluded). + +To confirm the same, type the follwoing command: + +ldd + +You should see that your application is using libcudart.so file in GPGPUSim directory. + Copy the contents of configs/QuadroFX5800/ or configs/GTX480/ to your application's working directory. These files configure the microarchitecture models to resemble the respective GPGPU architectures. @@ -348,7 +356,24 @@ identify any compile time or runtime errors that occur due to the code merging process. -** Debugging failing GPGPU-Sim Regressions ** +4. MISCELLANEOUS + +4.1 Speeding up the execution + +Some applications take several hours to execute on GPGPUSim. This is because the simulator has to dump the PTX, analyze them and get resource usage statistics. This can be avoided everytime we execute the program in the following way: + +Step 1: Execute the program by enabling “-save_embedded_ptx 1” in config file, execute the code and let cuobjdump command dump all necessary files. After this process, you will get 2 new files namely: _cuobjdump_complete_output_ and _1.ptx + +Step 2: Create new environment variables or include the below in your .bashrc file: + a. export PTX_SIM_USE_PTX_FILE=_1.ptx + b. export PTX_SIM_KERNELFILE=_1.ptx + c. export CUOBJDUMP_SIM_FILE=_cuobjdump_complete_output_ + +Step 3: Disable -save_embedded_ptx flag, execute the code again. This will skip the dumping by cuobjdump and directly goes to executing the program thus saving time. + + +4.2 Debugging failing GPGPU-Sim Regressions + Credits: Tor M Aamodt To debug failing GPGPU-Sim regression tests you need to run them locally. The fastest way to do this, assuming you are working with GPGPU-Sim versions more recent than the GPGPU-Sim dev branch circa March 28, 2018 (commit hash 2221d208a745a098a60b0d24c05007e92aaba092), is to install Docker. The instructions below were tested with Docker CE version 18.03 on Ubuntu and Mac OS. Docker will enable you to run the same set of regressions used by GPGPU-Sim when submitting a pull request to https://github.com/gpgpu-sim/gpgpu-sim_distribution and also allow you to log in and launch GPGPU-Sim in gdb so you can inspect failures. diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 0025c52..e53aaab 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -154,7 +154,9 @@ ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_in } else if ( op.is_local() ) { result.u64 = op.get_symbol()->get_address(); } else if ( op.is_function_address() ) { - result.u64 = (size_t)op.get_symbol()->get_pc(); + result.u64 = (size_t)op.get_symbol()->get_pc(); + } else if ( op.is_param_kernel()) { + result.u64 = op.get_symbol()->get_address(); } else { const char *name = op.name().c_str(); printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand type for %s\n", name ); diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 6731763..58d5f49 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -164,6 +164,7 @@ public: m_is_global = false; m_is_local = false; m_is_param_local = false; + m_is_param_kernel = false; m_is_tex = false; m_is_func_addr = false; m_reg_num_valid = false; @@ -177,6 +178,7 @@ public: if ( type ) m_is_global = type->get_key().is_global(); if ( type ) m_is_local = type->get_key().is_local(); if ( type ) m_is_param_local = type->get_key().is_param_local(); + if ( type ) m_is_param_kernel = type->get_key().is_param_kernel(); if ( type ) m_is_tex = type->get_key().is_tex(); if ( type ) m_is_func_addr = type->get_key().is_func_addr(); } @@ -227,6 +229,7 @@ public: bool is_global() const { return m_is_global;} bool is_local() const { return m_is_local;} bool is_param_local() const { return m_is_param_local; } + bool is_param_kernel() const { return m_is_param_kernel; } bool is_tex() const { return m_is_tex;} bool is_func_addr() const { return m_is_func_addr; } bool is_reg() const @@ -284,6 +287,7 @@ private: bool m_is_global; bool m_is_local; bool m_is_param_local; + bool m_is_param_kernel; bool m_is_tex; bool m_is_func_addr; unsigned m_reg_num; @@ -400,6 +404,8 @@ public: m_type = symbolic_t; } else if ( addr->is_param_local() ) { m_type = symbolic_t; + } else if ( addr->is_param_kernel() ) { + m_type = symbolic_t; } else if ( addr->is_tex() ) { m_type = symbolic_t; } else if ( addr->is_func_addr() ) { @@ -676,6 +682,13 @@ public: return m_value.m_symbolic->type()->get_key().is_param_local(); } + bool is_param_kernel() const + { + if ( m_type != symbolic_t ) + return false; + return m_value.m_symbolic->type()->get_key().is_param_kernel(); + } + bool is_vector() const { if ( m_vector) return true; -- cgit v1.3 From 5fd987fc8e551b823b0299e0cd3ce0d161fb84b6 Mon Sep 17 00:00:00 2001 From: Suchita Pati Date: Fri, 18 May 2018 20:28:02 -0500 Subject: ld operand in register space --- src/cuda-sim/instructions.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/cuda-sim/instructions.cc') diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index e53aaab..583a49a 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -2368,7 +2368,11 @@ void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand space = param_space_kernel; else if( ti.is_param_local() ) { space = param_space_local; - } else { + } + else if( ti.is_reg() ) { + space = param_space_kernel; + } + else { printf("GPGPU-Sim PTX: ERROR ** cannot resolve .param space for '%s'\n", s->name().c_str() ); abort(); } -- cgit v1.3 From a9f90b837af13e205886c19696f779e15e2499b0 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Wed, 16 May 2018 13:36:15 -0700 Subject: is_reg change --- .gitignore | 4 ++++ src/cuda-sim/instructions.cc | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'src/cuda-sim/instructions.cc') diff --git a/.gitignore b/.gitignore index 887b605..428abce 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,7 @@ cuobjdump_to_ptxplus/sass_parser.output build/* *.swp +*~ +cscope* +tags + diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 583a49a..86951ed 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -2366,7 +2366,7 @@ void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand type_info_key ti = t->get_key(); if( ti.is_param_kernel() ) space = param_space_kernel; - else if( ti.is_param_local() ) { + else if( ti.is_param_local() ) space = param_space_local; } else if( ti.is_reg() ) { -- cgit v1.3 From ff958f36689c9c217eb099326ceb8f70ed3ac447 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 22 May 2018 14:00:59 -0700 Subject: restructured texref maps to fix texture bug --- src/abstract_hardware_model.h | 36 +++++++++++++++++-------------- src/cuda-sim/cuda-sim.cc | 49 ++++++++++++++++++++++++++++--------------- src/cuda-sim/instructions.cc | 6 +++--- 3 files changed, 55 insertions(+), 36 deletions(-) (limited to 'src/cuda-sim/instructions.cc') diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index ab94ded..d0af1ea 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -65,6 +65,7 @@ enum FuncCache #include #include +#include typedef unsigned long long new_addr_type; typedef unsigned address_type; @@ -515,29 +516,31 @@ public: void gpgpu_ptx_sim_unbindTexture(const struct textureReference* texref); const char* gpgpu_ptx_sim_findNamefromTexture(const struct textureReference* texref); - const struct textureReference* get_texref(const std::string &texname) const + const struct textureReference* get_texref( const std::string &texname ) const { - std::map::const_iterator t=m_NameToTextureRef.find(texname); + std::map >::const_iterator t=m_NameToTextureRef.find(texname); assert( t != m_NameToTextureRef.end() ); - return t->second; + return *(t->second.begin()); } - const struct cudaArray* get_texarray( const struct textureReference *texref ) const + + const struct cudaArray* get_texarray( const std::string &texname ) const { - std::map::const_iterator t=m_TextureRefToCudaArray.find(texref); - assert(t != m_TextureRefToCudaArray.end()); + std::map::const_iterator t=m_NameToCudaArray.find(texname); + assert(t != m_NameToCudaArray.end()); return t->second; } - const struct textureInfo* get_texinfo( const struct textureReference *texref ) const + + const struct textureInfo* get_texinfo( const std::string &texname ) const { - std::map::const_iterator t=m_TextureRefToTexureInfo.find(texref); - assert(t != m_TextureRefToTexureInfo.end()); + std::map::const_iterator t=m_NameToTexureInfo.find(texname); + assert(t != m_NameToTexureInfo.end()); return t->second; } - const struct textureReferenceAttr* get_texattr( const struct textureReference *texref ) const + const struct textureReferenceAttr* get_texattr( const std::string &texname ) const { - std::map::const_iterator t=m_TextureRefToAttribute.find(texref); - assert(t != m_TextureRefToAttribute.end()); + std::map::const_iterator t=m_NameToAttribute.find(texname); + assert(t != m_NameToAttribute.end()); return t->second; } @@ -554,10 +557,11 @@ protected: unsigned long long m_dev_malloc; - std::map m_NameToTextureRef; - std::map m_TextureRefToCudaArray; - std::map m_TextureRefToTexureInfo; - std::map m_TextureRefToAttribute; + std::map > m_NameToTextureRef; + std::map m_TextureRefToName; + std::map m_NameToCudaArray; + std::map m_NameToTexureInfo; + std::map m_NameToAttribute; }; struct gpgpu_ptx_sim_info diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 6125422..05b6201 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -105,22 +105,36 @@ static address_type get_converge_point(address_type pc); void gpgpu_t::gpgpu_ptx_sim_bindNameToTexture(const char* name, const struct textureReference* texref, int dim, int readmode, int ext) { std::string texname(name); - m_NameToTextureRef[texname] = texref; + if (m_NameToTextureRef.find(texname)==m_NameToTextureRef.end()){ + m_NameToTextureRef[texname] = std::set(); + m_NameToTextureRef[texname].insert(texref); + }else{ + const struct textureReference* tr = *m_NameToTextureRef[texname].begin(); + assert(tr!=NULL); + //asserts that all texrefs in set have same fields + assert(tr->normalized==texref->normalized&& + tr->filterMode==texref->filterMode&& + tr->addressMode[0]==texref->addressMode[0]&& + tr->addressMode[1]==texref->addressMode[1]&& + tr->addressMode[2]==texref->addressMode[2]&& + tr->channelDesc.x==texref->channelDesc.x&& + tr->channelDesc.y==texref->channelDesc.y&& + tr->channelDesc.z==texref->channelDesc.z&& + tr->channelDesc.w==texref->channelDesc.w&& + tr->channelDesc.f==texref->channelDesc.f + ); + m_NameToTextureRef[texname].insert(texref); + } + m_TextureRefToName[texref] = texname; const textureReferenceAttr *texAttr = new textureReferenceAttr(texref, dim, (enum cudaTextureReadMode)readmode, ext); - m_TextureRefToAttribute[texref] = texAttr; + m_NameToAttribute[texname] = texAttr; } const char* gpgpu_t::gpgpu_ptx_sim_findNamefromTexture(const struct textureReference* texref) { - std::map::iterator itr = m_NameToTextureRef.begin(); - while (itr != m_NameToTextureRef.end()) { - if ((*itr).second == texref) { - const char *p = ((*itr).first).c_str(); - return p; - } - itr++; - } - return NULL; + std::map::const_iterator t=m_TextureRefToName.find(texref); + assert( t != m_TextureRefToName.end() ); + return t->second.c_str(); } unsigned int intLOGB2( unsigned int v ) { @@ -140,7 +154,8 @@ unsigned int intLOGB2( unsigned int v ) { void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* texref, const struct cudaArray* array) { - m_TextureRefToCudaArray[texref] = array; + std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); + m_NameToCudaArray[texname] = array; unsigned int texel_size_bits = array->desc.w + array->desc.x + array->desc.y + array->desc.z; unsigned int texel_size = texel_size_bits/8; unsigned int Tx, Ty; @@ -180,13 +195,14 @@ void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* te texInfo->Ty_numbits = intLOGB2(Ty); texInfo->texel_size = texel_size; texInfo->texel_size_numbits = intLOGB2(texel_size); - m_TextureRefToTexureInfo[texref] = texInfo; + m_NameToTexureInfo[texname] = texInfo; } void gpgpu_t::gpgpu_ptx_sim_unbindTexture(const struct textureReference* texref) { - m_TextureRefToCudaArray.erase(texref); - m_TextureRefToTexureInfo.erase(texref); + std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); + m_NameToCudaArray.erase(texname); + m_NameToTexureInfo.erase(texname); } unsigned g_assemble_code_next_pc=0; @@ -1246,8 +1262,7 @@ static unsigned get_tex_datasize( const ptx_instruction *pI, ptx_thread_info *th std::string texname = src1.name(); gpgpu_t *gpu = thread->get_gpu(); - const struct textureReference* texref = gpu->get_texref(texname); - const struct textureInfo* texInfo = gpu->get_texinfo(texref); + const struct textureInfo* texInfo = gpu->get_texinfo(texname); unsigned data_size = texInfo->texel_size; return data_size; diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 86951ed..d362231 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -4100,9 +4100,9 @@ void tex_impl( const ptx_instruction *pI, ptx_thread_info *thread ) gpgpu_t *gpu = thread->get_gpu(); const struct textureReference* texref = gpu->get_texref(texname); - const struct cudaArray* cuArray = gpu->get_texarray(texref); - const struct textureInfo* texInfo = gpu->get_texinfo(texref); - const struct textureReferenceAttr* texAttr = gpu->get_texattr(texref); + const struct cudaArray* cuArray = gpu->get_texarray(texname); + const struct textureInfo* texInfo = gpu->get_texinfo(texname); + const struct textureReferenceAttr* texAttr = gpu->get_texattr(texname); //assume always 2D f32 input //access array with src2 coordinates -- cgit v1.3 From 7e08538246f386a8e4ce0686d3f6eafff99f23cc Mon Sep 17 00:00:00 2001 From: Jonathan Date: Fri, 25 May 2018 09:32:49 -0700 Subject: fixed merge issue --- src/cuda-sim/instructions.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/cuda-sim/instructions.cc') diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index d362231..35d1782 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -2366,12 +2366,12 @@ void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand type_info_key ti = t->get_key(); if( ti.is_param_kernel() ) space = param_space_kernel; - else if( ti.is_param_local() ) + else if( ti.is_param_local() ) { space = param_space_local; } else if( ti.is_reg() ) { space = param_space_kernel; - } + } else { printf("GPGPU-Sim PTX: ERROR ** cannot resolve .param space for '%s'\n", s->name().c_str() ); abort(); -- cgit v1.3 From 73fea7152926dbc41cf008a4ed3402cc1feeefa7 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Mon, 4 Jun 2018 16:53:23 -0700 Subject: parses through all ptx files, TODO: need to impl dp4a --- libcuda/cuda_runtime_api.cc | 1 - src/cuda-sim/instructions.cc | 7 +++++++ src/cuda-sim/opcodes.def | 1 + src/cuda-sim/ptx.l | 1 + src/cuda-sim/ptx_loader.cc | 13 ------------- src/cuda-sim/ptx_parser.cc | 6 ++++++ 6 files changed, 15 insertions(+), 14 deletions(-) (limited to 'src/cuda-sim/instructions.cc') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index ece958e..d03caf7 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -1546,7 +1546,6 @@ void extract_ptx_files_using_cuobjdump(bool g_cdp_enabled){ void cuobjdumpParseBinary(unsigned int handle){ - if(fatbin_registered[handle]) return; fatbin_registered[handle] = true; CUctx_st *context = GPGPUSim_Context(); diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 35d1782..37438fa 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -2289,6 +2289,13 @@ void div_impl( const ptx_instruction *pI, ptx_thread_info *thread ) thread->set_operand_value(dst,data, i_type, thread,pI); } +void dp4a_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + printf("instruction not implemented yet"); + assert(0); + +} + void ex2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { ptx_reg_t src1_data, src2_data, data; diff --git a/src/cuda-sim/opcodes.def b/src/cuda-sim/opcodes.def index e1b1422..2dbfb78 100644 --- a/src/cuda-sim/opcodes.def +++ b/src/cuda-sim/opcodes.def @@ -60,6 +60,7 @@ OP_DEF(COS_OP,cos_impl,"cos",1,4) OP_DEF(CVT_OP,cvt_impl,"cvt",1,1) OP_DEF(CVTA_OP,cvta_impl,"cvta",1,1) OP_DEF(DIV_OP,div_impl,"div",1,1) +OP_DEF(DP4A_OP,dp4a_impl,"dp4a",1,1) OP_DEF(EX2_OP,ex2_impl,"ex2",1,4) OP_DEF(EXIT_OP,exit_impl,"exit",1,3) OP_DEF(FMA_OP,fma_impl,"fma",1,2) diff --git a/src/cuda-sim/ptx.l b/src/cuda-sim/ptx.l index 908c5be..3bb0d17 100644 --- a/src/cuda-sim/ptx.l +++ b/src/cuda-sim/ptx.l @@ -77,6 +77,7 @@ cos TC; ptx_lval.int_value = COS_OP; return OPCODE; cvt TC; ptx_lval.int_value = CVT_OP; return OPCODE; cvta TC; ptx_lval.int_value = CVTA_OP; return OPCODE; div TC; ptx_lval.int_value = DIV_OP; return OPCODE; +dp4a TC; ptx_lval.int_value = DP4A_OP; return OPCODE; ex2 TC; ptx_lval.int_value = EX2_OP; return OPCODE; exit TC; ptx_lval.int_value = EXIT_OP; return OPCODE; fma TC; ptx_lval.int_value = FMA_OP; return OPCODE; diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc index 8deafc6..0348af0 100644 --- a/src/cuda-sim/ptx_loader.cc +++ b/src/cuda-sim/ptx_loader.cc @@ -188,19 +188,6 @@ symbol_table *gpgpu_ptx_sim_load_ptx_from_string( const char *p, unsigned source symbol_table *gpgpu_ptx_sim_load_ptx_from_filename( const char *filename ) { symbol_table *symtab=init_parser(filename); - int errors = ptx_parse (); - if ( errors ) { - char fname[1024]; - snprintf(fname,1024,"_ptx_errors_XXXXXX"); - int fd=mkstemp(fname); - close(fd); - printf("GPGPU-Sim PTX: parser error detected, exiting... but first extracting .ptx to \"%s\"\n", fname); - FILE *ptxfile = fopen(fname,"w"); - fclose(ptxfile); - abort(); - exit(40); - } - printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n",filename); return symtab; } diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index a180da9..a51799a 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -32,6 +32,8 @@ extern int ptx_error( const char *s ); extern int ptx_lineno; +extern int ptx_parse(); +extern FILE *ptx_in; static const struct core_config *g_shader_core_config; void set_ptx_warp_size(const struct core_config * warp_size) @@ -135,6 +137,10 @@ symbol_table *init_parser( const char *ptx_filename ) g_ptx_token_decode[generic_space] = "generic_space"; g_ptx_token_decode[instruction_space] = "instruction_space"; + + ptx_in = fopen(ptx_filename, "r"); + ptx_parse(); + fclose(ptx_in); return g_global_symbol_table; } -- cgit v1.3 From f7b0d64c68f12d604e09aec8dbba569df354faf6 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 5 Jun 2018 12:58:08 -0700 Subject: parse all ptx and add to symbol table --- src/cuda-sim/instructions.cc | 2 +- src/cuda-sim/ptx_ir.cc | 2 +- src/cuda-sim/ptx_parser.cc | 69 ++++++++++++++++++++++---------------------- 3 files changed, 37 insertions(+), 36 deletions(-) (limited to 'src/cuda-sim/instructions.cc') diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 37438fa..08bf528 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -2291,7 +2291,7 @@ void div_impl( const ptx_instruction *pI, ptx_thread_info *thread ) void dp4a_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { - printf("instruction not implemented yet"); + printf("DP4A instruction not implemented yet"); assert(0); } diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index be25dbe..016c600 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -1031,7 +1031,7 @@ static std::list check_operands( int opcode, const std::list &operands ) { static int g_warn_literal_operands_two_type_inst; - if( (opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || (opcode == TEX_OP) ) { + if( (opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || (opcode == TEX_OP) || (opcode == DP4A_OP) ) { // just make sure these do not have have const operands... if( !g_warn_literal_operands_two_type_inst ) { std::list::const_iterator o; diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index a51799a..06ca870 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -107,6 +107,34 @@ void read_parser_environment_variables() } } +void init_directive_state() +{ + PTX_PARSE_DPRINTF("init_directive_state"); + g_space_spec=undefined_space; + g_ptr_spec=undefined_space; + g_scalar_type_spec=-1; + g_vector_spec=-1; + g_opcode=-1; + g_alignment_spec = -1; + g_extern_spec = 0; + g_scalar_type.clear(); + g_operands.clear(); + g_last_symbol = NULL; +} + +void init_instruction_state() +{ + PTX_PARSE_DPRINTF("init_instruction_state"); + g_pred = NULL; + g_neg_pred = 0; + g_pred_mod = -1; + g_label = NULL; + g_opcode = -1; + g_options.clear(); + g_return_var = operand_info(); + init_directive_state(); +} + symbol_table *init_parser( const char *ptx_filename ) { g_filename = strdup(ptx_filename); @@ -114,9 +142,9 @@ symbol_table *init_parser( const char *ptx_filename ) g_global_allfiles_symbol_table = new symbol_table("global_allfiles", 0, NULL); g_global_symbol_table = g_current_symbol_table = g_global_allfiles_symbol_table; } - else { - g_global_symbol_table = g_current_symbol_table = new symbol_table("global",0,g_global_allfiles_symbol_table); - } +// else { +// g_global_symbol_table = g_current_symbol_table = new symbol_table("global",0,g_global_allfiles_symbol_table); +// } ptx_lineno = 1; #define DEF(X,Y) g_ptx_token_decode[X] = Y; @@ -136,7 +164,8 @@ symbol_table *init_parser( const char *ptx_filename ) g_ptx_token_decode[global_space] = "global_space"; g_ptx_token_decode[generic_space] = "generic_space"; g_ptx_token_decode[instruction_space] = "instruction_space"; - + init_directive_state(); + init_instruction_state(); ptx_in = fopen(ptx_filename, "r"); ptx_parse(); @@ -144,34 +173,6 @@ symbol_table *init_parser( const char *ptx_filename ) return g_global_symbol_table; } -void init_directive_state() -{ - PTX_PARSE_DPRINTF("init_directive_state"); - g_space_spec=undefined_space; - g_ptr_spec=undefined_space; - g_scalar_type_spec=-1; - g_vector_spec=-1; - g_opcode=-1; - g_alignment_spec = -1; - g_extern_spec = 0; - g_scalar_type.clear(); - g_operands.clear(); - g_last_symbol = NULL; -} - -void init_instruction_state() -{ - PTX_PARSE_DPRINTF("init_instruction_state"); - g_pred = NULL; - g_neg_pred = 0; - g_pred_mod = -1; - g_label = NULL; - g_opcode = -1; - g_options.clear(); - g_return_var = operand_info(); - init_directive_state(); -} - static int g_entry_point; void start_function( int entry_point ) @@ -621,8 +622,8 @@ void add_scalar_type_spec( int type_spec ) g_scalar_type.push_back( type_spec ); if ( g_scalar_type.size() > 1 ) { parse_assert( (g_opcode == -1) || (g_opcode == CVT_OP) || (g_opcode == SET_OP) || (g_opcode == SLCT_OP) - || (g_opcode == TEX_OP), - "only cvt, set, slct, and tex can have more than one type specifier."); + || (g_opcode == TEX_OP)|| (g_opcode == DP4A_OP), + "only cvt, set, slct, tex, and dp4a can have more than one type specifier."); } g_scalar_type_spec = type_spec; } -- cgit v1.3 From 37dc5311a75548b848a33a3b7369ce4bee64b444 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Thu, 14 Jun 2018 20:15:52 -0700 Subject: bfe bug fix --- .gitignore | 5 +++++ src/cuda-sim/instructions.cc | 30 +++++++++++++++--------------- src/cuda-sim/ptx_ir.h | 2 +- 3 files changed, 21 insertions(+), 16 deletions(-) (limited to 'src/cuda-sim/instructions.cc') diff --git a/.gitignore b/.gitignore index 0e2a898..e4e1631 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,8 @@ cscope* tags regression.sh +#gcov +*.gcov +*.gcda +*.gcno + diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 08bf528..c77e4da 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -1353,45 +1353,45 @@ void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread ) const operand_info &src1 = pI->src1(); const operand_info &src2 = pI->src2(); const operand_info &src3 = pI->src3(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t src = thread->get_operand_value(src1, dst, i_type, thread, 1); ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + ptx_reg_t data; unsigned pos = b.u32 & 0xFF; unsigned len = c.u32 & 0xFF; - unsigned d = 0; switch (i_type) { case U32_TYPE: { unsigned mask; - d = a.u32 >> pos; + data.u32 = src.u32 >> pos; mask = 0xFFFFFFFF >> (32 - len); - d &= mask; + data.u32 &= mask; break; } case U64_TYPE: { unsigned long mask; - d = a.u64 >> pos; + data.u64 = src.u64 >> pos; mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - d &= mask; + data.u64 &= mask; break; } case S32_TYPE: { unsigned mask; unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (a.s32 >> min) & 0x1; - d = a.s32 >> pos; + unsigned sbit = len == 0 ? 0 : (src.s32 >> min) & 0x1; + data.s32 = src.s32 >> pos; if (sbit > 0) { mask = 0xFFFFFFFF << len; - d |= mask; + data.s32 |= mask; } else { mask = 0xFFFFFFFF >> (32 - len); - d &= mask; + data.s32 &= mask; } break; } @@ -1399,17 +1399,17 @@ void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { unsigned long mask; unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (a.s64 >> min) & 0x1; - d = a.s64 >> pos; + unsigned sbit = len == 0 ? 0 : (src.s64 >> min) & 0x1; + data.s64 = src.s64 >> pos; if (sbit > 0) { mask = 0xFFFFFFFFFFFFFFFF << len; - d |= mask; + data.s64 |= mask; } else { mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - d &= mask; + data.s64 &= mask; } break; } @@ -1418,7 +1418,7 @@ void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread ) abort(); return; } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } void bfi_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 58d5f49..5b68fcf 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -745,7 +745,7 @@ public: { ptx_reg_t result; switch ( m_type ) { - case int_t: result.s32 = m_value.m_int; break; + case int_t: result.s64 = m_value.m_int; break; case float_op_t: result.f32 = m_value.m_float; break; case double_op_t: result.f64 = m_value.m_double; break; case unsigned_t: result.u32 = m_value.m_unsigned; break; -- cgit v1.3 From 76a124e9186b9574858238d423b9c5ce715f2c32 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Thu, 21 Jun 2018 17:35:07 -0700 Subject: WIP adding support for PTX JIT and dumping params to cudaLaunches --- libcuda/cuda_runtime_api.cc | 168 ++++++++++++++++++++++++++++++++++++++++--- src/cuda-sim/cuda-sim.cc | 81 +++++++++++++++++++++ src/cuda-sim/cuda-sim.h | 1 + src/cuda-sim/instructions.cc | 1 + src/cuda-sim/ptx_ir.cc | 8 +++ src/cuda-sim/ptx_ir.h | 2 + 6 files changed, 252 insertions(+), 9 deletions(-) (limited to 'src/cuda-sim/instructions.cc') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index 08ee413..300fa28 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -126,6 +126,9 @@ #include "host_defines.h" #include "builtin_types.h" #include "driver_types.h" +#if (CUDART_VERSION >= 8000) +#include "cuda.h" +#endif #if (CUDART_VERSION < 8000) #include "__cudaFatFormat.h" #endif @@ -287,6 +290,34 @@ struct CUctx_st { } } + void register_hostFun_function( const char*hostFun, function_info* f){ + m_kernel_lookup[hostFun] = f; + } + + dim3 get_blockdim(const char *hostFun) + { + std::map::iterator i=m_hostFun_blockdim.find(hostFun); + assert( i != m_hostFun_blockdim.end() ); + return i->second; + } + + dim3 get_griddim(const char *hostFun) + { + std::map::iterator i=m_hostFun_griddim.find(hostFun); + assert( i != m_hostFun_griddim.end() ); + return i->second; + } + + void set_blockdim(const char *hostFun, dim3 dims) + { + m_hostFun_blockdim[hostFun] = dims; + } + + void set_griddim(const char *hostFun, dim3 dims) + { + m_hostFun_griddim[hostFun] = dims; + } + function_info *get_kernel(const char *hostFun) { std::map::iterator i=m_kernel_lookup.find(hostFun); @@ -298,6 +329,8 @@ private: _cuda_device_id *m_gpu; // selected gpu std::map m_code; // fat binary handle => global symbol table unsigned m_last_fat_cubin_handle; + std::map m_hostFun_blockdim; + std::map m_hostFun_griddim; std::map m_kernel_lookup; // unique id (CUDA app function address) => kernel entry point struct gpgpu_ptx_sim_info m_binary_info; @@ -1300,16 +1333,15 @@ int CUDARTAPI __cudaSynchronizeThreads(void**, void*) * * *******************************************************************************/ -#if (CUDART_VERSION >= 3010) +#if (CUDART_VERSION >= 3010 && CUDART_VERSION < 8000) typedef struct CUuuid_st { /**< CUDA definition of UUID */ char bytes[16]; } CUuuid; -/** - * CUDA UUID types - */ -// typedef __device_builtin__ struct CUuuid_st cudaUUID_t; +#endif + +#if (CUDART_VERSION >= 3010) __host__ cudaError_t CUDARTAPI cudaGetExportTable(const void **ppExportTable, const cudaUUID_t *pExportTableId) { @@ -1958,10 +1990,10 @@ void cuobjdumpParseBinary(unsigned int handle){ symtab = gpgpu_ptx_sim_load_ptx_from_filename( ptx_filename.c_str() ); } } - name_symtab[fname] = symtab; - context->add_binary(symtab, handle); - load_static_globals(symtab,STATIC_ALLOC_LIMIT,0xFFFFFFFF,context->get_device()->get_gpgpu()); - load_constants(symtab,STATIC_ALLOC_LIMIT,context->get_device()->get_gpgpu()); + name_symtab[fname] = symtab; + context->add_binary(symtab, handle); + load_static_globals(symtab,STATIC_ALLOC_LIMIT,0xFFFFFFFF,context->get_device()->get_gpgpu()); + load_constants(symtab,STATIC_ALLOC_LIMIT,context->get_device()->get_gpgpu()); return; #endif @@ -2602,5 +2634,123 @@ kernel_info_t *gpgpu_cuda_ptx_sim_init_grid( const char *hostFun, g_ptx_kernel_count++; fflush(stdout); + if(g_debug_execution >= 3){ + entry->debug_param(); + } + return result; } + +CUresult CUDAAPI cuLinkCreate(unsigned int numOptions, CUjit_option *options, void **optionValues, CUlinkState *stateOut) +{ + //currently do not support options or multiple CUlinkStates + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuLinkAddData(CUlinkState state, CUjitInputType type, void *data, size_t size, const char *name, + unsigned int numOptions, CUjit_option *options, void **optionValues) +{ + assert(type==CU_JIT_INPUT_PTX); + cuda_not_implemented(__my_func__,__LINE__); + return CUDA_ERROR_UNKNOWN; +} + +CUresult CUDAAPI cuLinkAddFile(CUlinkState state, CUjitInputType type, const char *path, + unsigned int numOptions, CUjit_option *options, void **optionValues) +{ + static bool addedFile = false; + if (addedFile){ + printf("GPGPU-Sim PTX: ERROR: cuLinkAddFile does not support multiple file"); + abort(); + } + + //blocking + assert(type==CU_JIT_INPUT_PTX); + CUctx_st *context = GPGPUSim_Context(); + char *file = getenv("PTX_JIT_PATH"); + if(file==NULL){ + printf("GPGPU-Sim PTX: ERROR: PTX_JIT_PATH has not been set"); + abort(); + } + strcat(file,path); + symbol_table *symtab = gpgpu_ptx_sim_load_ptx_from_filename( file ); + std::string fname(path); + name_symtab[fname] = symtab; + context->add_binary(symtab, 1); + load_static_globals(symtab,STATIC_ALLOC_LIMIT,0xFFFFFFFF,context->get_device()->get_gpgpu()); + load_constants(symtab,STATIC_ALLOC_LIMIT,context->get_device()->get_gpgpu()); + addedFile = true; + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuLinkComplete(CUlinkState state, void **cubinOut, size_t *sizeOut) +{ + //all cuLink* function are implemented to block until completion so nothing to do here + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuLinkDestroy(CUlinkState state) +{ + //currently do not support options or multiple CUlinkStates + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuModuleLoadData(CUmodule *module, const void *image) +{ + //Currently do not support multiple modules + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuModuleGetFunction(CUfunction *hfunc, CUmodule hmod, const char *name) +{ + CUctx_st* context = GPGPUSim_Context(); + std::string key(name); + //only support one file + assert(name_symtab.size()==1); + symbol_table* symtab = name_symtab.begin()->second; + function_info* f = symtab->lookup_function( std::string(name) ); + //just need to add given pointer to map for cudaLaunch + context->register_hostFun_function( (const char*) hfunc, f); + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuModuleUnload(CUmodule hmod) +{ + //Currently do not support multiple modules + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuFuncSetBlockShape(CUfunction hfunc, int x, int y, int z) +{ + CUctx_st* context = GPGPUSim_Context(); + dim3 dims(x,y,z); + context->set_blockdim((const char *)hfunc, dims); + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuParamSetSize(CUfunction hfunc, unsigned int numbytes) +{ + //Nothing to do + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuParamSetv(CUfunction hfunc, int offset, void *ptr, unsigned int numbytes) +{ + cuda_not_implemented(__my_func__,__LINE__); + return CUDA_ERROR_UNKNOWN; +} + +CUresult CUDAAPI cuLaunchGrid(CUfunction f, int grid_width, int grid_height) +{ + cuda_not_implemented(__my_func__,__LINE__); + return CUDA_ERROR_UNKNOWN; + + CUctx_st* context = GPGPUSim_Context(); + const char *hostFun = (const char*) f; + dim3 dims(grid_width,grid_height,1); + context->set_griddim((const char *)f, dims); + cudaConfigureCall(context->get_griddim(hostFun), context->get_blockdim(hostFun), 0, NULL); + + cudaLaunch(hostFun); + return CUDA_SUCCESS; +} diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 34368ce..6875edd 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1226,6 +1226,87 @@ void function_info::list_param( FILE *fout ) const fflush(fout); } +void function_info::debug_param( ) const +{ + char filename[] = "params.txt"; + char buff[1024]; + snprintf(buff,1024,"c++filt %s > %s", get_name().c_str(), filename); + system(buff); + FILE *fp = fopen(filename, "r"); + fgets(buff, 1024, fp); + fclose(fp); + + std::string fn(buff); + size_t pos1, pos2; + pos1 = fn.find("("); + pos2 = fn.find(")"); + assert(pos2>pos1&&pos1>0); + strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); + printf("params: %s\n", buff); + char *tok; + std::vector params; + tok = strtok(buff, ","); + while(tok!=NULL){ + std::string param(tok); + param.erase(0, param.find_first_not_of(" ")); + param.erase(param.find_last_not_of(" ")+1); + params.push_back(param); + tok = strtok(NULL, ","); + } + for (auto const& it : params){ + std::cout<::const_iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + const param_info &p = i->second; + std::string name = p.get_name(); + param_t param_value = p.get_value(); + if(params[i->first].find("const")!=std::string::npos){ + fprintf(fout, "Input: "); + } else { + fprintf(fout, "Input/output: "); + } + + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); + fprintf(fout, "%s: %#08x, ", name.c_str(), param_addr); + + if(params[i->first].find("int")!=std::string::npos){ + size_t len = param_value.size/sizeof(int); + int val[len]; + memcpy((void*) val, param_value.pdata+param_value.offset, param_value.size); + fprintf(fout, "val (int) = "); + for (unsigned i = 0; ifirst].find("float")!=std::string::npos){ + size_t len = param_value.size/sizeof(float); + float val[len]; + memcpy((void*) val, param_value.pdata+param_value.offset, param_value.size); + fprintf(fout, "val (float) = "); + for (unsigned i = 0; i bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc) { diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h index 958daba..9049a84 100644 --- a/src/cuda-sim/cuda-sim.h +++ b/src/cuda-sim/cuda-sim.h @@ -32,6 +32,7 @@ #include"../gpgpu-sim/shader.h" #include #include +#include #include #include"ptx_sim.h" diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index c77e4da..034a7b9 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -2753,6 +2753,7 @@ void mov_impl( const ptx_instruction *pI, ptx_thread_info *thread ) const operand_info &dst = pI->dst(); const operand_info &src1 = pI->src1(); unsigned i_type = pI->get_type(); + assert( src1.is_param_local() == 0 ); if( (src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && (i_type != BB128_TYPE) && (i_type != FF64_TYPE) ) { // pack or unpack operation diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index 016c600..482b9e0 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -257,6 +257,14 @@ bool symbol_table::add_function_decl( const char *name, int entry_point, functio return prior_decl; } +function_info *symbol_table::lookup_function( std::string name ) +{ + std::string key = std::string(name); + std::map::iterator it = m_function_info_lookup.find(key); + assert ( it != m_function_info_lookup.end() ); + return it->second; +} + type_info *symbol_table::add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ) { if( space_spec == param_space_unclassified ) diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 5b68fcf..341f9b7 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -313,6 +313,7 @@ public: symbol *add_variable( const char *identifier, const type_info *type, unsigned size, const char *filename, unsigned line ); void add_function( function_info *func, const char *filename, unsigned linenumber ); bool add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **symbol_table ); + function_info *lookup_function(std::string name); type_info *add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ); type_info *add_type( function_info *func ); type_info *get_array_type( type_info *base_type, unsigned array_dim ); @@ -1256,6 +1257,7 @@ public: void finalize( memory_space *param_mem ); void param_to_shared( memory_space *shared_mem, symbol_table *symtab ); void list_param( FILE *fout ) const; + void debug_param() const; const struct gpgpu_ptx_sim_info* get_kernel_info () const { -- cgit v1.3 From d072fa72c6ffce7f34520d24c8cb285e0d5b92b2 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Fri, 13 Jul 2018 10:46:47 -0700 Subject: Add more print statements to debug api calls Fixed bug with 32bit rem Fixed bug with launching kernels in ptxjitplus Removed extraneous files Cleaned up more pointers Added documentation for ptxjitplus --- debug_tools/WatchYourStep/ptxjitplus/Makefile | 5 +- debug_tools/WatchYourStep/ptxjitplus/ptxinst.cpp | 14 --- debug_tools/WatchYourStep/ptxjitplus/ptxinst.h | 14 --- .../WatchYourStep/ptxjitplus/ptxjitplus.cpp | 103 ++++++++++++--------- libcuda/cuda_runtime_api.cc | 36 +++++++ src/cuda-sim/cuda-sim.cc | 4 +- src/cuda-sim/instructions.cc | 16 +++- 7 files changed, 113 insertions(+), 79 deletions(-) delete mode 100644 debug_tools/WatchYourStep/ptxjitplus/ptxinst.cpp delete mode 100644 debug_tools/WatchYourStep/ptxjitplus/ptxinst.h (limited to 'src/cuda-sim/instructions.cc') diff --git a/debug_tools/WatchYourStep/ptxjitplus/Makefile b/debug_tools/WatchYourStep/ptxjitplus/Makefile index d571567..b273ac2 100644 --- a/debug_tools/WatchYourStep/ptxjitplus/Makefile +++ b/debug_tools/WatchYourStep/ptxjitplus/Makefile @@ -315,14 +315,11 @@ endif ptxjitplus.o:ptxjitplus.cpp $(EXEC) $(NVCC) $(INCLUDES) --cudart shared $(ALL_CCFLAGS) $(GENCODE_FLAGS) -g -o $@ -c $< -ptxjitplus: ptxjitplus.o ptxinst.o +ptxjitplus: ptxjitplus.o $(EXEC) $(NVCC) $(ALL_LDFLAGS) --cudart shared $(GENCODE_FLAGS) -g -o $@ $+ $(LIBRARIES) $(EXEC) mkdir -p ./bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE) $(EXEC) cp $@ ./bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE) -ptxinst.o:ptxinst.cpp - $(EXEC) $(NVCC) $(INCLUDES) --cudart shared $(ALL_CCFLAGS) $(GENCODE_FLAGS) -g -o $@ -c $< - run: build $(EXEC) ./ptxjitplus diff --git a/debug_tools/WatchYourStep/ptxjitplus/ptxinst.cpp b/debug_tools/WatchYourStep/ptxjitplus/ptxinst.cpp deleted file mode 100644 index 6b39eb1..0000000 --- a/debug_tools/WatchYourStep/ptxjitplus/ptxinst.cpp +++ /dev/null @@ -1,14 +0,0 @@ -/* ptxinst.cpp - * Jonathan Lew - * University of British Columbia - */ -#include "ptxinst.h" - -void* instrument_ptx_from_function(std::string function, std::string path) -{ - return NULL; -} -void* instrument_ptx_from_string(std::string ptxcode) -{ - return NULL; -} diff --git a/debug_tools/WatchYourStep/ptxjitplus/ptxinst.h b/debug_tools/WatchYourStep/ptxjitplus/ptxinst.h deleted file mode 100644 index de2595e..0000000 --- a/debug_tools/WatchYourStep/ptxjitplus/ptxinst.h +++ /dev/null @@ -1,14 +0,0 @@ -/* ptxinst.h - * Jonathan Lew - * University of British Columbia - */ - -#ifndef _PTXINST_H_ -#define _PTXINST_H_ - -#include - -void* instrument_ptx_from_function(std::string function, std::string path); -void* instrument_ptx_from_string(std::string ptxcode); - -#endif diff --git a/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp b/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp index 68964d8..b7f9f2d 100644 --- a/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp +++ b/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp @@ -1,5 +1,4 @@ -/** - * Copyright 1993-2015 NVIDIA Corporation. All rights reserved. +/** Copyright 1993-2015 NVIDIA Corporation. All rights reserved. * * Please refer to the NVIDIA end user license agreement (EULA) associated * with this source code for terms and conditions that govern your use of @@ -7,9 +6,6 @@ * this software and related documentation outside the terms of the EULA * is strictly prohibited. * - */ - -/* * This sample uses the Driver API to just-in-time compile (JIT) a Kernel from PTX code. * Additionally, this sample demonstrates the seamless interoperability capability of CUDA runtime * Runtime and CUDA Driver API calls. @@ -17,6 +13,32 @@ * */ +/** + * Modified by: Jonathan Lew + * PTX JIT PLUS + * + * ********** + * User Guide + * ********** + * + * Welcome to WatchYourStep, a debugging tool that allows you launch individual + * kernels using parameters captured from cudaLaunch and outputs the values in + * the arrays from the kernel. It allows you to watch each step you program takes, + * kernel by kernel. + * + * 1. Set environment variables to create params.config* and ptx.config* files. + * a)export PTX_SIM_DEBUG=4 + * b)export PTX_JIT_PATH=[path to this file] + * c)export WYS_EXEC_PATH=[path to executable (program to debug)] + * d)export WYS_EXEC_NAME=[name of executable (program to debug)] + * e)Make sure all GPGPU-Sim path variables are set (see GPGPU-Sim documentation) + * 2. Run executable (program to debug) using GPGPU-Sim + * 3. export PTX_SIM_DEBUG=[less than 4 to not dump config files again] + * 4-1. Run one kernel at a time: export WYS_LAUNCH_NUM=[kernel to launch] and compile ptxjitplus and run ptxjitplus + * 4-2. Run all kernels: compile and run ". launchkernels 0 [max number of kernels]" in terminal + * 5. Find output in ../data/wys.out* where * is the launch number + */ + // System includes #include #include @@ -34,12 +56,12 @@ // sample include #include "ptxjitplus.h" -#include "ptxinst.h" -const char *sSDKname = "PTX Just In Time (JIT) Compilation (no-qatest)"; +const char *sSDKname = "PTX Just In Time (JIT) Compilation Plus"; char *wys_exec_path; char *wys_exec_name; char *wys_launch_num; +bool gpgpusim = false; dim3 gridDim, blockDim; std::string kernelName; @@ -116,6 +138,10 @@ void ptxJIT(int argc, char **argv, CUmodule *phModule, CUfunction *phKernel, CUl void initializeData(std::vector& param_data, std::vector< std::pair >& param_info) { + char *gpgpusim_env = getenv("GPGPUSIM_SETUP_ENVIRONMENT_WAS_RUN"); + if (gpgpusim_env!=NULL&&gpgpusim_env[0] == '1'){ + gpgpusim=true; + } wys_exec_path = getenv("WYS_EXEC_PATH"); assert(wys_exec_path!=NULL); wys_exec_name = getenv("WYS_EXEC_NAME"); @@ -150,32 +176,19 @@ void initializeData(std::vector& param_data, std::vector< std::p err = fscanf(fin, "%lu : ", &len); info.first = len; assert( err==1 ); - //printf("%lu : ", len); unsigned char* params = (unsigned char*) malloc(len*sizeof(unsigned char)); for (size_t i=0; isecond[j]); - //} - //fprintf(fout, "\n"); - //fflush(fout); - //fclose(fout); } int main(int argc, char **argv) @@ -249,20 +262,28 @@ int main(int argc, char **argv) checkCudaErrors(cudaFree(d_tmp)); //maps param number to pointer to device data - std::map< size_t, unsigned char* > m_device_data; + std::map< size_t, void* > m_device_data; + std::map< size_t, void* > m_cleanup; + void * paramKernels[param_data.size()]; //Initialize param_data for kernel int paramOffset = 0; for( size_t i = 0; i m_output_data; - for(std::map< size_t, unsigned char* >::iterator i = m_device_data.begin(); i!=m_device_data.end(); i++){ + for(std::map< size_t, void* >::iterator i = m_device_data.begin(); i!=m_device_data.end(); i++){ unsigned char *h_data = 0; if ((h_data = (unsigned char *)malloc(param_info[i->first].first)) == NULL) { @@ -294,40 +315,32 @@ int main(int argc, char **argv) for(std::map< size_t, unsigned char* >::iterator i = m_output_data.begin(); i!=m_output_data.end(); i++){ fprintf(fout, "param %zu: size = %zu, data = ", i->first,param_info[i->first].first); for (size_t j = 0; jfirst].first; j++){ - fprintf(fout, " %u", i->second[j]); - if (j&&(!(j%20))){ + if (!(j%24)){ fprintf(fout, "\n"); } + fprintf(fout, " %u", i->second[j]); } fprintf(fout, "\n"); } fflush(fout); fclose(fout); -// int* h_data = (int*) m_output_data[0]; -// // Check the result -// bool dataGood = true; -// -// for (unsigned int i = 0 ; dataGood && i < nBlocks * nThreads ; i++) -// { -// if (h_data[i] != (int)i) -// { -// std::cerr << "Error at " << i << std::endl; -// dataGood = false; -// } -// } -// if(dataGood){ -// std::cout<<"OK!"<::iterator i = m_device_data.begin(); i!=m_device_data.end(); i++){ + for(std::map< size_t, void* >::iterator i = m_device_data.begin(); i!=m_device_data.end(); i++){ if (i->second){ checkCudaErrors(cudaFree(i->second)); i->second = 0; } } + for(std::map< size_t, void* >::iterator i = m_cleanup.begin(); i!=m_cleanup.end(); i++){ + if (i->second){ + free(i->second); + i->second = 0; + } + } + for(std::map< size_t, unsigned char* >::iterator i = m_output_data.begin(); i!=m_output_data.begin(); i++){ if (i->second) { @@ -342,5 +355,5 @@ int main(int argc, char **argv) hModule = 0; } - //return dataGood ? EXIT_SUCCESS : EXIT_FAILURE; + return 0; } diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index 68b4017..866fa3b 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -2347,11 +2347,17 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin ) void __cudaUnregisterFatBinary(void **fatCubinHandle) { + if(g_debug_execution >= 3){ + announce_call(__my_func__); + } ; } cudaError_t cudaDeviceReset ( void ) { // Should reset the simulated GPU + if(g_debug_execution >= 3){ + announce_call(__my_func__); + } return g_last_cudaError = cudaSuccess; } cudaError_t CUDARTAPI cudaDeviceSynchronize(void){ @@ -2397,6 +2403,9 @@ extern void __cudaRegisterVar( int constant, int global ) { + if(g_debug_execution >= 3){ + announce_call(__my_func__); + } printf("GPGPU-Sim PTX: __cudaRegisterVar: hostVar = %p; deviceAddress = %s; deviceName = %s\n", hostVar, deviceAddress, deviceName); printf("GPGPU-Sim PTX: __cudaRegisterVar: Registering const memory space of %d bytes\n", size); if(GPGPUSim_Context()->get_device()->get_gpgpu()->get_config().use_cuobjdump()) @@ -2415,6 +2424,9 @@ void __cudaRegisterShared( void **devicePtr ) { + if(g_debug_execution >= 3){ + announce_call(__my_func__); + } // we don't do anything here printf("GPGPU-Sim PTX: __cudaRegisterShared\n" ); } @@ -2444,6 +2456,9 @@ void __cudaRegisterTexture( int ext ) //passes in a newly created textureReference { + if(g_debug_execution >= 3){ + announce_call(__my_func__); + } std::string devStr (deviceName); #if (CUDART_VERSION > 4020) if (devStr.size() > 2 && devStr.data()[0] == ':' && devStr.data()[1] == ':') @@ -2465,6 +2480,9 @@ typedef unsigned long GLuint; cudaError_t cudaGLRegisterBufferObject(GLuint bufferObj) { + if(g_debug_execution >= 3){ + announce_call(__my_func__); + } printf("GPGPU-Sim PTX: Execution warning: ignoring call to \"%s\"\n", __my_func__ ); return g_last_cudaError = cudaSuccess; } @@ -2481,6 +2499,9 @@ glbmap_entry_t* g_glbmap = NULL; cudaError_t cudaGLMapBufferObject(void** devPtr, GLuint bufferObj) { + if(g_debug_execution >= 3){ + announce_call(__my_func__); + } #ifdef OPENGL_SUPPORT GLint buffer_size=0; CUctx_st* ctx = GPGPUSim_Context(); @@ -2534,6 +2555,9 @@ cudaError_t cudaGLMapBufferObject(void** devPtr, GLuint bufferObj) cudaError_t cudaGLUnmapBufferObject(GLuint bufferObj) { + if(g_debug_execution >= 3){ + announce_call(__my_func__); + } #ifdef OPENGL_SUPPORT glbmap_entry_t *p = g_glbmap; while ( p && p->m_bufferObj != bufferObj ) @@ -2558,6 +2582,9 @@ cudaError_t cudaGLUnmapBufferObject(GLuint bufferObj) cudaError_t cudaGLUnregisterBufferObject(GLuint bufferObj) { + if(g_debug_execution >= 3){ + announce_call(__my_func__); + } printf("GPGPU-Sim PTX: Execution warning: ignoring call to \"%s\"\n", __my_func__ ); return g_last_cudaError = cudaSuccess; } @@ -2783,6 +2810,9 @@ extern FILE *ptxinfo_in; static int load_static_globals( symbol_table *symtab, unsigned min_gaddr, unsigned max_gaddr, gpgpu_t *gpu ) { + if(g_debug_execution >= 3){ + announce_call(__my_func__); + } printf( "GPGPU-Sim PTX: loading globals with explicit initializers... \n" ); fflush(stdout); int ng_bytes=0; @@ -2819,6 +2849,9 @@ static int load_static_globals( symbol_table *symtab, unsigned min_gaddr, unsign static int load_constants( symbol_table *symtab, addr_t min_gaddr, gpgpu_t *gpu ) { + if(g_debug_execution >= 3){ + announce_call(__my_func__); + } printf( "GPGPU-Sim PTX: loading constants with explicit initializers... " ); fflush(stdout); int nc_bytes = 0; @@ -2866,6 +2899,9 @@ kernel_info_t *gpgpu_cuda_ptx_sim_init_grid( const char *hostFun, struct dim3 blockDim, CUctx_st* context ) { + if(g_debug_execution >= 3){ + announce_call(__my_func__); + } function_info *entry = context->get_kernel(hostFun); kernel_info_t *result = new kernel_info_t(gridDim,blockDim,entry); if( entry == NULL ) { diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index df8d806..d2f096f 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1309,7 +1309,9 @@ void function_info::ptx_jit_config(std::map mallocPt do{ fprintf(fout, "%s", buff); fgets(buff, 1024, fin); - assert(!feof(fin)); + if(feof(fin)){ + break; + } } while(strstr(buff, "entry")==NULL); fclose(fin); diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 034a7b9..9e18772 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -3190,7 +3190,21 @@ void rem_impl( const ptx_instruction *pI, ptx_thread_info *thread ) src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - data.u64 = src1_data.u64 % src2_data.u64; + switch ( i_type ) { + case S32_TYPE: + data.s32 = src1_data.s32 % src2_data.s32; + break; + case S64_TYPE: + data.s64 = src1_data.s64 % src2_data.s64; + break; + case U32_TYPE: + data.u32 = src1_data.u32 % src2_data.u32; + break; + case U64_TYPE: + data.u64 = src1_data.u64 % src2_data.u64; + break; + default: assert(0); break; + } thread->set_operand_value(dst,data, i_type, thread, pI); } -- cgit v1.3 From 742919652e8c4bd3ac33248175bc7fca0e419227 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Mon, 16 Jul 2018 14:28:32 -0700 Subject: fp32 to fp16 and fp16 to fp32 cvt --- src/cuda-sim/half.hpp | 3067 ++++++++++++++++++++++++++++++++++++++++++ src/cuda-sim/instructions.cc | 103 +- 2 files changed, 3121 insertions(+), 49 deletions(-) create mode 100644 src/cuda-sim/half.hpp (limited to 'src/cuda-sim/instructions.cc') diff --git a/src/cuda-sim/half.hpp b/src/cuda-sim/half.hpp new file mode 100644 index 0000000..8f1a8eb --- /dev/null +++ b/src/cuda-sim/half.hpp @@ -0,0 +1,3067 @@ +// half - IEEE 754-based half-precision floating point library. +// +// Copyright (c) 2012-2017 Christian Rau +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, +// modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +// Version 1.12.0 + +/// \file +/// Main header file for half precision functionality. + +#ifndef HALF_HALF_HPP +#define HALF_HALF_HPP + +/// Combined gcc version number. +#define HALF_GNUC_VERSION (__GNUC__*100+__GNUC_MINOR__) + +//check C++11 language features +#if defined(__clang__) //clang + #if __has_feature(cxx_static_assert) && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) + #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 + #endif + #if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR) + #define HALF_ENABLE_CPP11_CONSTEXPR 1 + #endif + #if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT) + #define HALF_ENABLE_CPP11_NOEXCEPT 1 + #endif + #if __has_feature(cxx_user_literals) && !defined(HALF_ENABLE_CPP11_USER_LITERALS) + #define HALF_ENABLE_CPP11_USER_LITERALS 1 + #endif + #if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && !defined(HALF_ENABLE_CPP11_LONG_LONG) + #define HALF_ENABLE_CPP11_LONG_LONG 1 + #endif +/*#elif defined(__INTEL_COMPILER) //Intel C++ + #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) ???????? + #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 + #endif + #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) ???????? + #define HALF_ENABLE_CPP11_CONSTEXPR 1 + #endif + #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) ???????? + #define HALF_ENABLE_CPP11_NOEXCEPT 1 + #endif + #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_LONG_LONG) ???????? + #define HALF_ENABLE_CPP11_LONG_LONG 1 + #endif*/ +#elif defined(__GNUC__) //gcc + #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L + #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) + #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 + #endif + #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) + #define HALF_ENABLE_CPP11_CONSTEXPR 1 + #endif + #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) + #define HALF_ENABLE_CPP11_NOEXCEPT 1 + #endif + #if HALF_GNUC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) + #define HALF_ENABLE_CPP11_USER_LITERALS 1 + #endif + #if !defined(HALF_ENABLE_CPP11_LONG_LONG) + #define HALF_ENABLE_CPP11_LONG_LONG 1 + #endif + #endif +#elif defined(_MSC_VER) //Visual C++ + #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) + #define HALF_ENABLE_CPP11_CONSTEXPR 1 + #endif + #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) + #define HALF_ENABLE_CPP11_NOEXCEPT 1 + #endif + #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) + #define HALF_ENABLE_CPP11_USER_LITERALS 1 + #endif + #if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) + #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 + #endif + #if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG) + #define HALF_ENABLE_CPP11_LONG_LONG 1 + #endif + #define HALF_POP_WARNINGS 1 + #pragma warning(push) + #pragma warning(disable : 4099 4127 4146) //struct vs class, constant in if, negative unsigned +#endif + +//check C++11 library features +#include +#if defined(_LIBCPP_VERSION) //libc++ + #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 + #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS + #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 + #endif + #ifndef HALF_ENABLE_CPP11_CSTDINT + #define HALF_ENABLE_CPP11_CSTDINT 1 + #endif + #ifndef HALF_ENABLE_CPP11_CMATH + #define HALF_ENABLE_CPP11_CMATH 1 + #endif + #ifndef HALF_ENABLE_CPP11_HASH + #define HALF_ENABLE_CPP11_HASH 1 + #endif + #endif +#elif defined(__GLIBCXX__) //libstdc++ + #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 + #ifdef __clang__ + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) + #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 + #endif + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT) + #define HALF_ENABLE_CPP11_CSTDINT 1 + #endif + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH) + #define HALF_ENABLE_CPP11_CMATH 1 + #endif + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH) + #define HALF_ENABLE_CPP11_HASH 1 + #endif + #else + #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT) + #define HALF_ENABLE_CPP11_CSTDINT 1 + #endif + #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH) + #define HALF_ENABLE_CPP11_CMATH 1 + #endif + #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH) + #define HALF_ENABLE_CPP11_HASH 1 + #endif + #endif + #endif +#elif defined(_CPPLIB_VER) //Dinkumware/Visual C++ + #if _CPPLIB_VER >= 520 + #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS + #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 + #endif + #ifndef HALF_ENABLE_CPP11_CSTDINT + #define HALF_ENABLE_CPP11_CSTDINT 1 + #endif + #ifndef HALF_ENABLE_CPP11_HASH + #define HALF_ENABLE_CPP11_HASH 1 + #endif + #endif + #if _CPPLIB_VER >= 610 + #ifndef HALF_ENABLE_CPP11_CMATH + #define HALF_ENABLE_CPP11_CMATH 1 + #endif + #endif +#endif +#undef HALF_GNUC_VERSION + +//support constexpr +#if HALF_ENABLE_CPP11_CONSTEXPR + #define HALF_CONSTEXPR constexpr + #define HALF_CONSTEXPR_CONST constexpr +#else + #define HALF_CONSTEXPR + #define HALF_CONSTEXPR_CONST const +#endif + +//support noexcept +#if HALF_ENABLE_CPP11_NOEXCEPT + #define HALF_NOEXCEPT noexcept + #define HALF_NOTHROW noexcept +#else + #define HALF_NOEXCEPT + #define HALF_NOTHROW throw() +#endif + +#include +#include +#include +#include +#include +#include +#if HALF_ENABLE_CPP11_TYPE_TRAITS + #include +#endif +#if HALF_ENABLE_CPP11_CSTDINT + #include +#endif +#if HALF_ENABLE_CPP11_HASH + #include +#endif + + +/// Default rounding mode. +/// This specifies the rounding mode used for all conversions between [half](\ref half_float::half)s and `float`s as well as +/// for the half_cast() if not specifying a rounding mode explicitly. It can be redefined (before including half.hpp) to one +/// of the standard rounding modes using their respective constants or the equivalent values of `std::float_round_style`: +/// +/// `std::float_round_style` | value | rounding +/// ---------------------------------|-------|------------------------- +/// `std::round_indeterminate` | -1 | fastest (default) +/// `std::round_toward_zero` | 0 | toward zero +/// `std::round_to_nearest` | 1 | to nearest +/// `std::round_toward_infinity` | 2 | toward positive infinity +/// `std::round_toward_neg_infinity` | 3 | toward negative infinity +/// +/// By default this is set to `-1` (`std::round_indeterminate`), which uses truncation (round toward zero, but with overflows +/// set to infinity) and is the fastest rounding mode possible. It can even be set to `std::numeric_limits::round_style` +/// to synchronize the rounding mode with that of the underlying single-precision implementation. +#ifndef HALF_ROUND_STYLE + #define HALF_ROUND_STYLE -1 // = std::round_indeterminate +#endif + +/// Tie-breaking behaviour for round to nearest. +/// This specifies if ties in round to nearest should be resolved by rounding to the nearest even value. By default this is +/// defined to `0` resulting in the faster but slightly more biased behaviour of rounding away from zero in half-way cases (and +/// thus equal to the round() function), but can be redefined to `1` (before including half.hpp) if more IEEE-conformant +/// behaviour is needed. +#ifndef HALF_ROUND_TIES_TO_EVEN + #define HALF_ROUND_TIES_TO_EVEN 0 // ties away from zero +#endif + +/// Value signaling overflow. +/// In correspondence with `HUGE_VAL[F|L]` from `` this symbol expands to a positive value signaling the overflow of an +/// operation, in particular it just evaluates to positive infinity. +#define HUGE_VALH std::numeric_limits::infinity() + +/// Fast half-precision fma function. +/// This symbol is only defined if the fma() function generally executes as fast as, or faster than, a separate +/// half-precision multiplication followed by an addition. Due to the internal single-precision implementation of all +/// arithmetic operations, this is in fact always the case. +#define FP_FAST_FMAH 1 + +#ifndef FP_ILOGB0 + #define FP_ILOGB0 INT_MIN +#endif +#ifndef FP_ILOGBNAN + #define FP_ILOGBNAN INT_MAX +#endif +#ifndef FP_SUBNORMAL + #define FP_SUBNORMAL 0 +#endif +#ifndef FP_ZERO + #define FP_ZERO 1 +#endif +#ifndef FP_NAN + #define FP_NAN 2 +#endif +#ifndef FP_INFINITE + #define FP_INFINITE 3 +#endif +#ifndef FP_NORMAL + #define FP_NORMAL 4 +#endif + + +/// Main namespace for half precision functionality. +/// This namespace contains all the functionality provided by the library. +namespace half_float +{ + class half; + +#if HALF_ENABLE_CPP11_USER_LITERALS + /// Library-defined half-precision literals. + /// Import this namespace to enable half-precision floating point literals: + /// ~~~~{.cpp} + /// using namespace half_float::literal; + /// half_float::half = 4.2_h; + /// ~~~~ + namespace literal + { + half operator"" _h(long double); + } +#endif + + /// \internal + /// \brief Implementation details. + namespace detail + { + #if HALF_ENABLE_CPP11_TYPE_TRAITS + /// Conditional type. + template struct conditional : std::conditional {}; + + /// Helper for tag dispatching. + template struct bool_type : std::integral_constant {}; + using std::true_type; + using std::false_type; + + /// Type traits for floating point types. + template struct is_float : std::is_floating_point {}; + #else + /// Conditional type. + template struct conditional { typedef T type; }; + template struct conditional { typedef F type; }; + + /// Helper for tag dispatching. + template struct bool_type {}; + typedef bool_type true_type; + typedef bool_type false_type; + + /// Type traits for floating point types. + template struct is_float : false_type {}; + template struct is_float : is_float {}; + template struct is_float : is_float {}; + template struct is_float : is_float {}; + template<> struct is_float : true_type {}; + template<> struct is_float : true_type {}; + template<> struct is_float : true_type {}; + #endif + + /// Type traits for floating point bits. + template struct bits { typedef unsigned char type; }; + template struct bits : bits {}; + template struct bits : bits {}; + template struct bits : bits {}; + + #if HALF_ENABLE_CPP11_CSTDINT + /// Unsigned integer of (at least) 16 bits width. + typedef std::uint_least16_t uint16; + + /// Unsigned integer of (at least) 32 bits width. + template<> struct bits { typedef std::uint_least32_t type; }; + + /// Unsigned integer of (at least) 64 bits width. + template<> struct bits { typedef std::uint_least64_t type; }; + #else + /// Unsigned integer of (at least) 16 bits width. + typedef unsigned short uint16; + + /// Unsigned integer of (at least) 32 bits width. + template<> struct bits : conditional::digits>=32,unsigned int,unsigned long> {}; + + #if HALF_ENABLE_CPP11_LONG_LONG + /// Unsigned integer of (at least) 64 bits width. + template<> struct bits : conditional::digits>=64,unsigned long,unsigned long long> {}; + #else + /// Unsigned integer of (at least) 64 bits width. + template<> struct bits { typedef unsigned long type; }; + #endif + #endif + + /// Tag type for binary construction. + struct binary_t {}; + + /// Tag for binary construction. + HALF_CONSTEXPR_CONST binary_t binary = binary_t(); + + /// Temporary half-precision expression. + /// This class represents a half-precision expression which just stores a single-precision value internally. + struct expr + { + /// Conversion constructor. + /// \param f single-precision value to convert + explicit HALF_CONSTEXPR expr(float f) HALF_NOEXCEPT : value_(f) {} + + /// Conversion to single-precision. + /// \return single precision value representing expression value + HALF_CONSTEXPR operator float() const HALF_NOEXCEPT { return value_; } + + private: + /// Internal expression value stored in single-precision. + float value_; + }; + + /// SFINAE helper for generic half-precision functions. + /// This class template has to be specialized for each valid combination of argument types to provide a corresponding + /// `type` member equivalent to \a T. + /// \tparam T type to return + template struct enable {}; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + template struct enable { typedef T type; }; + + /// Return type for specialized generic 2-argument half-precision functions. + /// This class template has to be specialized for each valid combination of argument types to provide a corresponding + /// `type` member denoting the appropriate return type. + /// \tparam T first argument type + /// \tparam U first argument type + template struct result : enable {}; + template<> struct result { typedef half type; }; + + /// \name Classification helpers + /// \{ + + /// Check for infinity. + /// \tparam T argument type (builtin floating point type) + /// \param arg value to query + /// \retval true if infinity + /// \retval false else + template bool builtin_isinf(T arg) + { + #if HALF_ENABLE_CPP11_CMATH + return std::isinf(arg); + #elif defined(_MSC_VER) + return !::_finite(static_cast(arg)) && !::_isnan(static_cast(arg)); + #else + return arg == std::numeric_limits::infinity() || arg == -std::numeric_limits::infinity(); + #endif + } + + /// Check for NaN. + /// \tparam T argument type (builtin floating point type) + /// \param arg value to query + /// \retval true if not a number + /// \retval false else + template bool builtin_isnan(T arg) + { + #if HALF_ENABLE_CPP11_CMATH + return std::isnan(arg); + #elif defined(_MSC_VER) + return ::_isnan(static_cast(arg)) != 0; + #else + return arg != arg; + #endif + } + + /// Check sign. + /// \tparam T argument type (builtin floating point type) + /// \param arg value to query + /// \retval true if signbit set + /// \retval false else + template bool builtin_signbit(T arg) + { + #if HALF_ENABLE_CPP11_CMATH + return std::signbit(arg); + #else + return arg < T() || (arg == T() && T(1)/arg < T()); + #endif + } + + /// \} + /// \name Conversion + /// \{ + + /// Convert IEEE single-precision to half-precision. + /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \param value single-precision value + /// \return binary representation of half-precision value + template uint16 float2half_impl(float value, true_type) + { + typedef bits::type uint32; + uint32 bits;// = *reinterpret_cast(&value); //violating strict aliasing! + std::memcpy(&bits, &value, sizeof(float)); +/* uint16 hbits = (bits>>16) & 0x8000; + bits &= 0x7FFFFFFF; + int exp = bits >> 23; + if(exp == 255) + return hbits | 0x7C00 | (0x3FF&-static_cast((bits&0x7FFFFF)!=0)); + if(exp > 142) + { + if(R == std::round_toward_infinity) + return hbits | 0x7C00 - (hbits>>15); + if(R == std::round_toward_neg_infinity) + return hbits | 0x7BFF + (hbits>>15); + return hbits | 0x7BFF + (R!=std::round_toward_zero); + } + int g, s; + if(exp > 112) + { + g = (bits>>12) & 1; + s = (bits&0xFFF) != 0; + hbits |= ((exp-112)<<10) | ((bits>>13)&0x3FF); + } + else if(exp > 101) + { + int i = 125 - exp; + bits = (bits&0x7FFFFF) | 0x800000; + g = (bits>>i) & 1; + s = (bits&((1L<> (i+1); + } + else + { + g = 0; + s = bits != 0; + } + if(R == std::round_to_nearest) + #if HALF_ROUND_TIES_TO_EVEN + hbits += g & (s|hbits); + #else + hbits += g; + #endif + else if(R == std::round_toward_infinity) + hbits += ~(hbits>>15) & (s|g); + else if(R == std::round_toward_neg_infinity) + hbits += (hbits>>15) & (g|s); +*/ static const uint16 base_table[512] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100, + 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00, 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00, + 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100, + 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00, + 0xC000, 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00, 0xF000, 0xF400, 0xF800, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00 }; + static const unsigned char shift_table[512] = { + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13 }; + uint16 hbits = base_table[bits>>23] + static_cast((bits&0x7FFFFF)>>shift_table[bits>>23]); + if(R == std::round_to_nearest) + hbits += (((bits&0x7FFFFF)>>(shift_table[bits>>23]-1))|(((bits>>23)&0xFF)==102)) & ((hbits&0x7C00)!=0x7C00) + #if HALF_ROUND_TIES_TO_EVEN + & (((((static_cast(1)<<(shift_table[bits>>23]-1))-1)&bits)!=0)|hbits) + #endif + ; + else if(R == std::round_toward_zero) + hbits -= ((hbits&0x7FFF)==0x7C00) & ~shift_table[bits>>23]; + else if(R == std::round_toward_infinity) + hbits += ((((bits&0x7FFFFF&((static_cast(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=102)& + ((bits>>23)!=0)))&(hbits<0x7C00)) - ((hbits==0xFC00)&((bits>>23)!=511)); + else if(R == std::round_toward_neg_infinity) + hbits += ((((bits&0x7FFFFF&((static_cast(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=358)& + ((bits>>23)!=256)))&(hbits<0xFC00)&(hbits>>15)) - ((hbits==0x7C00)&((bits>>23)!=255)); + return hbits; + } + + /// Convert IEEE double-precision to half-precision. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \param value double-precision value + /// \return binary representation of half-precision value + template uint16 float2half_impl(double value, true_type) + { + typedef bits::type uint32; + typedef bits::type uint64; + uint64 bits;// = *reinterpret_cast(&value); //violating strict aliasing! + std::memcpy(&bits, &value, sizeof(double)); + uint32 hi = bits >> 32, lo = bits & 0xFFFFFFFF; + uint16 hbits = (hi>>16) & 0x8000; + hi &= 0x7FFFFFFF; + int exp = hi >> 20; + if(exp == 2047) + return hbits | 0x7C00 | (0x3FF&-static_cast((bits&0xFFFFFFFFFFFFF)!=0)); + if(exp > 1038) + { + if(R == std::round_toward_infinity) + return hbits | 0x7C00 - (hbits>>15); + if(R == std::round_toward_neg_infinity) + return hbits | 0x7BFF + (hbits>>15); + return hbits | 0x7BFF + (R!=std::round_toward_zero); + } + int g, s = lo != 0; + if(exp > 1008) + { + g = (hi>>9) & 1; + s |= (hi&0x1FF) != 0; + hbits |= ((exp-1008)<<10) | ((hi>>10)&0x3FF); + } + else if(exp > 997) + { + int i = 1018 - exp; + hi = (hi&0xFFFFF) | 0x100000; + g = (hi>>i) & 1; + s |= (hi&((1L<> (i+1); + } + else + { + g = 0; + s |= hi != 0; + } + if(R == std::round_to_nearest) + #if HALF_ROUND_TIES_TO_EVEN + hbits += g & (s|hbits); + #else + hbits += g; + #endif + else if(R == std::round_toward_infinity) + hbits += ~(hbits>>15) & (s|g); + else if(R == std::round_toward_neg_infinity) + hbits += (hbits>>15) & (g|s); + return hbits; + } + + /// Convert non-IEEE floating point to half-precision. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \tparam T source type (builtin floating point type) + /// \param value floating point value + /// \return binary representation of half-precision value + template uint16 float2half_impl(T value, ...) + { + uint16 hbits = static_cast(builtin_signbit(value)) << 15; + if(value == T()) + return hbits; + if(builtin_isnan(value)) + return hbits | 0x7FFF; + if(builtin_isinf(value)) + return hbits | 0x7C00; + int exp; + std::frexp(value, &exp); + if(exp > 16) + { + if(R == std::round_toward_infinity) + return hbits | 0x7C00 - (hbits>>15); + else if(R == std::round_toward_neg_infinity) + return hbits | 0x7BFF + (hbits>>15); + return hbits | 0x7BFF + (R!=std::round_toward_zero); + } + if(exp < -13) + value = std::ldexp(value, 24); + else + { + value = std::ldexp(value, 11-exp); + hbits |= ((exp+13)<<10); + } + T ival, frac = std::modf(value, &ival); + hbits += static_cast(std::abs(static_cast(ival))); + if(R == std::round_to_nearest) + { + frac = std::abs(frac); + #if HALF_ROUND_TIES_TO_EVEN + hbits += (frac>T(0.5)) | ((frac==T(0.5))&hbits); + #else + hbits += frac >= T(0.5); + #endif + } + else if(R == std::round_toward_infinity) + hbits += frac > T(); + else if(R == std::round_toward_neg_infinity) + hbits += frac < T(); + return hbits; + } + + /// Convert floating point to half-precision. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \tparam T source type (builtin floating point type) + /// \param value floating point value + /// \return binary representation of half-precision value + template uint16 float2half(T value) + { + return float2half_impl(value, bool_type::is_iec559&&sizeof(typename bits::type)==sizeof(T)>()); + } + + /// Convert integer to half-precision floating point. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \tparam S `true` if value negative, `false` else + /// \tparam T type to convert (builtin integer type) + /// \param value non-negative integral value + /// \return binary representation of half-precision value + template uint16 int2half_impl(T value) + { + #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_integral::value, "int to half conversion only supports builtin integer types"); + #endif + if(S) + value = -value; + uint16 bits = S << 15; + if(value > 0xFFFF) + { + if(R == std::round_toward_infinity) + bits |= 0x7C00 - S; + else if(R == std::round_toward_neg_infinity) + bits |= 0x7BFF + S; + else + bits |= 0x7BFF + (R!=std::round_toward_zero); + } + else if(value) + { + unsigned int m = value, exp = 24; + for(; m<0x400; m<<=1,--exp) ; + for(; m>0x7FF; m>>=1,++exp) ; + bits |= (exp<<10) + m; + if(exp > 24) + { + if(R == std::round_to_nearest) + bits += (value>>(exp-25)) & 1 + #if HALF_ROUND_TIES_TO_EVEN + & (((((1<<(exp-25))-1)&value)!=0)|bits) + #endif + ; + else if(R == std::round_toward_infinity) + bits += ((value&((1<<(exp-24))-1))!=0) & !S; + else if(R == std::round_toward_neg_infinity) + bits += ((value&((1<<(exp-24))-1))!=0) & S; + } + } + return bits; + } + + /// Convert integer to half-precision floating point. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \tparam T type to convert (builtin integer type) + /// \param value integral value + /// \return binary representation of half-precision value + template uint16 int2half(T value) + { + return (value<0) ? int2half_impl(value) : int2half_impl(value); + } + + /// Convert half-precision to IEEE single-precision. + /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). + /// \param value binary representation of half-precision value + /// \return single-precision value + inline float half2float_impl(uint16 value, float, true_type) + { + typedef bits::type uint32; +/* uint32 bits = static_cast(value&0x8000) << 16; + int abs = value & 0x7FFF; + if(abs) + { + bits |= 0x38000000 << static_cast(abs>=0x7C00); + for(; abs<0x400; abs<<=1,bits-=0x800000) ; + bits += static_cast(abs) << 13; + } +*/ static const uint32 mantissa_table[2048] = { + 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000, 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000, 0x35700000, + 0x35800000, 0x35880000, 0x35900000, 0x35980000, 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, 0x35F00000, 0x35F80000, + 0x36000000, 0x36040000, 0x36080000, 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000, 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000, + 0x36400000, 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000, 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, 0x36700000, 0x36740000, 0x36780000, 0x367C0000, + 0x36800000, 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000, 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369A0000, 0x369C0000, 0x369E0000, + 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000, + 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000, 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000, + 0x36E00000, 0x36E20000, 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000, 0x36FC0000, 0x36FE0000, + 0x37000000, 0x37010000, 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000, 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000, + 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000, + 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000, 0x372F0000, + 0x37300000, 0x37310000, 0x37320000, 0x37330000, 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, 0x373E0000, 0x373F0000, + 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000, + 0x37500000, 0x37510000, 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000, + 0x37600000, 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000, + 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000, 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000, + 0x37800000, 0x37808000, 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, 0x37870000, 0x37878000, + 0x37880000, 0x37888000, 0x37890000, 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, 0x378F8000, + 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, 0x37960000, 0x37968000, 0x37970000, 0x37978000, + 0x37980000, 0x37988000, 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000, 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000, + 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000, + 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000, 0x37AB0000, 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000, 0x37AF0000, 0x37AF8000, + 0x37B00000, 0x37B08000, 0x37B10000, 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, 0x37B68000, 0x37B70000, 0x37B78000, + 0x37B80000, 0x37B88000, 0x37B90000, 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000, 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000, + 0x37C00000, 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000, + 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, 0x37CF0000, 0x37CF8000, + 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000, 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000, + 0x37D80000, 0x37D88000, 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000, 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000, + 0x37E00000, 0x37E08000, 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000, 0x37E70000, 0x37E78000, + 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000, 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000, + 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000, + 0x37F80000, 0x37F88000, 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, 0x37FF0000, 0x37FF8000, + 0x38000000, 0x38004000, 0x38008000, 0x3800C000, 0x38010000, 0x38014000, 0x38018000, 0x3801C000, 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000, 0x38034000, 0x38038000, 0x3803C000, + 0x38040000, 0x38044000, 0x38048000, 0x3804C000, 0x38050000, 0x38054000, 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, 0x38070000, 0x38074000, 0x38078000, 0x3807C000, + 0x38080000, 0x38084000, 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000, 0x380A0000, 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000, + 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000, 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, 0x380FC000, + 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, 0x38114000, 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000, 0x38130000, 0x38134000, 0x38138000, 0x3813C000, + 0x38140000, 0x38144000, 0x38148000, 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, 0x38174000, 0x38178000, 0x3817C000, + 0x38180000, 0x38184000, 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000, 0x3819C000, 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000, + 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000, 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000, + 0x38200000, 0x38204000, 0x38208000, 0x3820C000, 0x38210000, 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000, 0x38228000, 0x3822C000, 0x38230000, 0x38234000, 0x38238000, 0x3823C000, + 0x38240000, 0x38244000, 0x38248000, 0x3824C000, 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000, + 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000, 0x38298000, 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000, 0x382B4000, 0x382B8000, 0x382BC000, + 0x382C0000, 0x382C4000, 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000, + 0x38300000, 0x38304000, 0x38308000, 0x3830C000, 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000, 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000, + 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000, 0x38358000, 0x3835C000, 0x38360000, 0x38364000, 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000, 0x3837C000, + 0x38380000, 0x38384000, 0x38388000, 0x3838C000, 0x38390000, 0x38394000, 0x38398000, 0x3839C000, 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, 0x383B8000, 0x383BC000, + 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000, 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000, 0x383E8000, 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000, + 0x38400000, 0x38404000, 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000, 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, 0x38430000, 0x38434000, 0x38438000, 0x3843C000, + 0x38440000, 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, 0x3845C000, 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000, 0x38478000, 0x3847C000, + 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000, 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000, + 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000, + 0x38500000, 0x38504000, 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000, 0x38538000, 0x3853C000, + 0x38540000, 0x38544000, 0x38548000, 0x3854C000, 0x38550000, 0x38554000, 0x38558000, 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000, 0x38570000, 0x38574000, 0x38578000, 0x3857C000, + 0x38580000, 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000, + 0x385C0000, 0x385C4000, 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000, 0x385FC000, + 0x38600000, 0x38604000, 0x38608000, 0x3860C000, 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000, 0x38638000, 0x3863C000, + 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, 0x38654000, 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000, 0x38670000, 0x38674000, 0x38678000, 0x3867C000, + 0x38680000, 0x38684000, 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000, 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000, + 0x386C0000, 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000, + 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000, 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000, 0x38730000, 0x38734000, 0x38738000, 0x3873C000, + 0x38740000, 0x38744000, 0x38748000, 0x3874C000, 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000, 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, 0x38778000, 0x3877C000, + 0x38780000, 0x38784000, 0x38788000, 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000, + 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000, 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000, + 0x38000000, 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800A000, 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000, + 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000, + 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, 0x38050000, 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000, 0x3805C000, 0x3805E000, + 0x38060000, 0x38062000, 0x38064000, 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807A000, 0x3807C000, 0x3807E000, + 0x38080000, 0x38082000, 0x38084000, 0x38086000, 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000, 0x38094000, 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000, + 0x380A0000, 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000, + 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, 0x380CE000, 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, 0x380DC000, 0x380DE000, + 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000, 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000, + 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000, + 0x38120000, 0x38122000, 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000, 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000, 0x3813C000, 0x3813E000, + 0x38140000, 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000, 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000, 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000, + 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, 0x38176000, 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000, + 0x38180000, 0x38182000, 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, 0x3819C000, 0x3819E000, + 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, 0x381BC000, 0x381BE000, + 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000, + 0x381E0000, 0x381E2000, 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000, + 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, 0x3821E000, + 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000, 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823A000, 0x3823C000, 0x3823E000, + 0x38240000, 0x38242000, 0x38244000, 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825A000, 0x3825C000, 0x3825E000, + 0x38260000, 0x38262000, 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, 0x38272000, 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000, + 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000, + 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000, 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, 0x382BC000, 0x382BE000, + 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000, 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000, + 0x382E0000, 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000, 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000, 0x382FA000, 0x382FC000, 0x382FE000, + 0x38300000, 0x38302000, 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831A000, 0x3831C000, 0x3831E000, + 0x38320000, 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000, 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000, + 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000, + 0x38360000, 0x38362000, 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, 0x3837C000, 0x3837E000, + 0x38380000, 0x38382000, 0x38384000, 0x38386000, 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000, + 0x383A0000, 0x383A2000, 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000, + 0x383C0000, 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000, 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, 0x383DE000, + 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000, 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, 0x383FE000, + 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000, + 0x38420000, 0x38422000, 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000, 0x3843C000, 0x3843E000, + 0x38440000, 0x38442000, 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000, 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000, + 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000, + 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000, 0x3849E000, + 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, 0x384BE000, + 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000, 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000, 0x384DA000, 0x384DC000, 0x384DE000, + 0x384E0000, 0x384E2000, 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000, + 0x38500000, 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000, + 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000, 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, 0x38538000, 0x3853A000, 0x3853C000, 0x3853E000, + 0x38540000, 0x38542000, 0x38544000, 0x38546000, 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, 0x3855C000, 0x3855E000, + 0x38560000, 0x38562000, 0x38564000, 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000, + 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000, 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000, + 0x385A0000, 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, 0x385BA000, 0x385BC000, 0x385BE000, + 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000, + 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000, 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000, + 0x38600000, 0x38602000, 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000, 0x3861C000, 0x3861E000, + 0x38620000, 0x38622000, 0x38624000, 0x38626000, 0x38628000, 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000, + 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000, + 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000, 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000, 0x3867E000, + 0x38680000, 0x38682000, 0x38684000, 0x38686000, 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869A000, 0x3869C000, 0x3869E000, + 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000, + 0x386C0000, 0x386C2000, 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000, 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000, + 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000, 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000, + 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000, + 0x38720000, 0x38722000, 0x38724000, 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, 0x3873C000, 0x3873E000, + 0x38740000, 0x38742000, 0x38744000, 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, 0x3875E000, + 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000, 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000, + 0x38780000, 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000, + 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000, + 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000, 0x387CC000, 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000, 0x387DC000, 0x387DE000, + 0x387E0000, 0x387E2000, 0x387E4000, 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, 0x387FA000, 0x387FC000, 0x387FE000 }; + static const uint32 exponent_table[64] = { + 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, 0x06000000, 0x06800000, 0x07000000, 0x07800000, + 0x08000000, 0x08800000, 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, 0x0F000000, 0x47800000, + 0x80000000, 0x80800000, 0x81000000, 0x81800000, 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000, + 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000 }; + static const unsigned short offset_table[64] = { + 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, + 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024 }; + uint32 bits = mantissa_table[offset_table[value>>10]+(value&0x3FF)] + exponent_table[value>>10]; +// return *reinterpret_cast(&bits); //violating strict aliasing! + float out; + std::memcpy(&out, &bits, sizeof(float)); + return out; + } + + /// Convert half-precision to IEEE double-precision. + /// \param value binary representation of half-precision value + /// \return double-precision value + inline double half2float_impl(uint16 value, double, true_type) + { + typedef bits::type uint32; + typedef bits::type uint64; + uint32 hi = static_cast(value&0x8000) << 16; + int abs = value & 0x7FFF; + if(abs) + { + hi |= 0x3F000000 << static_cast(abs>=0x7C00); + for(; abs<0x400; abs<<=1,hi-=0x100000) ; + hi += static_cast(abs) << 10; + } + uint64 bits = static_cast(hi) << 32; +// return *reinterpret_cast(&bits); //violating strict aliasing! + double out; + std::memcpy(&out, &bits, sizeof(double)); + return out; + } + + /// Convert half-precision to non-IEEE floating point. + /// \tparam T type to convert to (builtin integer type) + /// \param value binary representation of half-precision value + /// \return floating point value + template T half2float_impl(uint16 value, T, ...) + { + T out; + int abs = value & 0x7FFF; + if(abs > 0x7C00) + out = std::numeric_limits::has_quiet_NaN ? std::numeric_limits::quiet_NaN() : T(); + else if(abs == 0x7C00) + out = std::numeric_limits::has_infinity ? std::numeric_limits::infinity() : std::numeric_limits::max(); + else if(abs > 0x3FF) + out = std::ldexp(static_cast((abs&0x3FF)|0x400), (abs>>10)-25); + else + out = std::ldexp(static_cast(abs), -24); + return (value&0x8000) ? -out : out; + } + + /// Convert half-precision to floating point. + /// \tparam T type to convert to (builtin integer type) + /// \param value binary representation of half-precision value + /// \return floating point value + template T half2float(uint16 value) + { + return half2float_impl(value, T(), bool_type::is_iec559&&sizeof(typename bits::type)==sizeof(T)>()); + } + + /// Convert half-precision floating point to integer. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \tparam E `true` for round to even, `false` for round away from zero + /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) + /// \param value binary representation of half-precision value + /// \return integral value + template T half2int_impl(uint16 value) + { + #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_integral::value, "half to int conversion only supports builtin integer types"); + #endif + unsigned int e = value & 0x7FFF; + if(e >= 0x7C00) + return (value&0x8000) ? std::numeric_limits::min() : std::numeric_limits::max(); + if(e < 0x3800) + { + if(R == std::round_toward_infinity) + return T(~(value>>15)&(e!=0)); + else if(R == std::round_toward_neg_infinity) + return -T(value>0x8000); + return T(); + } + unsigned int m = (value&0x3FF) | 0x400; + e >>= 10; + if(e < 25) + { + if(R == std::round_to_nearest) + m += (1<<(24-e)) - (~(m>>(25-e))&E); + else if(R == std::round_toward_infinity) + m += ((value>>15)-1) & ((1<<(25-e))-1U); + else if(R == std::round_toward_neg_infinity) + m += -(value>>15) & ((1<<(25-e))-1U); + m >>= 25 - e; + } + else + m <<= e - 25; + return (value&0x8000) ? -static_cast(m) : static_cast(m); + } + + /// Convert half-precision floating point to integer. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) + /// \param value binary representation of half-precision value + /// \return integral value + template T half2int(uint16 value) { return half2int_impl(value); } + + /// Convert half-precision floating point to integer using round-to-nearest-away-from-zero. + /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) + /// \param value binary representation of half-precision value + /// \return integral value + template T half2int_up(uint16 value) { return half2int_impl(value); } + + /// Round half-precision number to nearest integer value. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \tparam E `true` for round to even, `false` for round away from zero + /// \param value binary representation of half-precision value + /// \return half-precision bits for nearest integral value + template uint16 round_half_impl(uint16 value) + { + unsigned int e = value & 0x7FFF; + uint16 result = value; + if(e < 0x3C00) + { + result &= 0x8000; + if(R == std::round_to_nearest) + result |= 0x3C00U & -(e>=(0x3800+E)); + else if(R == std::round_toward_infinity) + result |= 0x3C00U & -(~(value>>15)&(e!=0)); + else if(R == std::round_toward_neg_infinity) + result |= 0x3C00U & -(value>0x8000); + } + else if(e < 0x6400) + { + e = 25 - (e>>10); + unsigned int mask = (1<>e)&E); + else if(R == std::round_toward_infinity) + result += mask & ((value>>15)-1); + else if(R == std::round_toward_neg_infinity) + result += mask & -(value>>15); + result &= ~mask; + } + return result; + } + + /// Round half-precision number to nearest integer value. + /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding + /// \param value binary representation of half-precision value + /// \return half-precision bits for nearest integral value + template uint16 round_half(uint16 value) { return round_half_impl(value); } + + /// Round half-precision number to nearest integer value using round-to-nearest-away-from-zero. + /// \param value binary representation of half-precision value + /// \return half-precision bits for nearest integral value + inline uint16 round_half_up(uint16 value) { return round_half_impl(value); } + /// \} + + struct functions; + template struct unary_specialized; + template struct binary_specialized; + template struct half_caster; + } + + /// Half-precision floating point type. + /// This class implements an IEEE-conformant half-precision floating point type with the usual arithmetic operators and + /// conversions. It is implicitly convertible to single-precision floating point, which makes artihmetic expressions and + /// functions with mixed-type operands to be of the most precise operand type. Additionally all arithmetic operations + /// (and many mathematical functions) are carried out in single-precision internally. All conversions from single- to + /// half-precision are done using the library's default rounding mode, but temporary results inside chained arithmetic + /// expressions are kept in single-precision as long as possible (while of course still maintaining a strong half-precision type). + /// + /// According to the C++98/03 definition, the half type is not a POD type. But according to C++11's less strict and + /// extended definitions it is both a standard layout type and a trivially copyable type (even if not a POD type), which + /// means it can be standard-conformantly copied using raw binary copies. But in this context some more words about the + /// actual size of the type. Although the half is representing an IEEE 16-bit type, it does not neccessarily have to be of + /// exactly 16-bits size. But on any reasonable implementation the actual binary representation of this type will most + /// probably not ivolve any additional "magic" or padding beyond the simple binary representation of the underlying 16-bit + /// IEEE number, even if not strictly guaranteed by the standard. But even then it only has an actual size of 16 bits if + /// your C++ implementation supports an unsigned integer type of exactly 16 bits width. But this should be the case on + /// nearly any reasonable platform. + /// + /// So if your C++ implementation is not totally exotic or imposes special alignment requirements, it is a reasonable + /// assumption that the data of a half is just comprised of the 2 bytes of the underlying IEEE representation. + class half + { + friend struct detail::functions; + friend struct detail::unary_specialized; + friend struct detail::binary_specialized; + template friend struct detail::half_caster; + friend class std::numeric_limits; + #if HALF_ENABLE_CPP11_HASH + friend struct std::hash; + #endif + #if HALF_ENABLE_CPP11_USER_LITERALS + friend half literal::operator"" _h(long double); + #endif + + public: + /// Default constructor. + /// This initializes the half to 0. Although this does not match the builtin types' default-initialization semantics + /// and may be less efficient than no initialization, it is needed to provide proper value-initialization semantics. + HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {} + + /// Copy constructor. + /// \tparam T type of concrete half expression + /// \param rhs half expression to copy from + half(detail::expr rhs) : data_(detail::float2half(static_cast(rhs))) {} + + /// Conversion constructor. + /// \param rhs float to convert + explicit half(float rhs) : data_(detail::float2half(rhs)) {} + + /// Conversion to single-precision. + /// \return single precision value representing expression value + operator float() const { return detail::half2float(data_); } + + /// Assignment operator. + /// \tparam T type of concrete half expression + /// \param rhs half expression to copy from + /// \return reference to this half + half& operator=(detail::expr rhs) { return *this = static_cast(rhs); } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to add + /// \return reference to this half + template typename detail::enable::type operator+=(T rhs) { return *this += static_cast(rhs); } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to subtract + /// \return reference to this half + template typename detail::enable::type operator-=(T rhs) { return *this -= static_cast(rhs); } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to multiply with + /// \return reference to this half + template typename detail::enable::type operator*=(T rhs) { return *this *= static_cast(rhs); } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to divide by + /// \return reference to this half + template typename detail::enable::type operator/=(T rhs) { return *this /= static_cast(rhs); } + + /// Assignment operator. + /// \param rhs single-precision value to copy from + /// \return reference to this half + half& operator=(float rhs) { data_ = detail::float2half(rhs); return *this; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to add + /// \return reference to this half + half& operator+=(float rhs) { data_ = detail::float2half(detail::half2float(data_)+rhs); return *this; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to subtract + /// \return reference to this half + half& operator-=(float rhs) { data_ = detail::float2half(detail::half2float(data_)-rhs); return *this; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to multiply with + /// \return reference to this half + half& operator*=(float rhs) { data_ = detail::float2half(detail::half2float(data_)*rhs); return *this; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to divide by + /// \return reference to this half + half& operator/=(float rhs) { data_ = detail::float2half(detail::half2float(data_)/rhs); return *this; } + + /// Prefix increment. + /// \return incremented half value + half& operator++() { return *this += 1.0f; } + + /// Prefix decrement. + /// \return decremented half value + half& operator--() { return *this -= 1.0f; } + + /// Postfix increment. + /// \return non-incremented half value + half operator++(int) { half out(*this); ++*this; return out; } + + /// Postfix decrement. + /// \return non-decremented half value + half operator--(int) { half out(*this); --*this; return out; } + + private: + /// Rounding mode to use + static const std::float_round_style round_style = (std::float_round_style)(HALF_ROUND_STYLE); + + /// Constructor. + /// \param bits binary representation to set half to + HALF_CONSTEXPR half(detail::binary_t, detail::uint16 bits) HALF_NOEXCEPT : data_(bits) {} + + /// Internal binary representation + detail::uint16 data_; + }; + +#if HALF_ENABLE_CPP11_USER_LITERALS + namespace literal + { + /// Half literal. + /// While this returns an actual half-precision value, half literals can unfortunately not be constant expressions due + /// to rather involved conversions. + /// \param value literal value + /// \return half with given value (if representable) + inline half operator"" _h(long double value) { return half(detail::binary, detail::float2half(value)); } + } +#endif + + namespace detail + { + /// Wrapper implementing unspecialized half-precision functions. + struct functions + { + /// Addition implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision sum stored in single-precision + static expr plus(float x, float y) { return expr(x+y); } + + /// Subtraction implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision difference stored in single-precision + static expr minus(float x, float y) { return expr(x-y); } + + /// Multiplication implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision product stored in single-precision + static expr multiplies(float x, float y) { return expr(x*y); } + + /// Division implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision quotient stored in single-precision + static expr divides(float x, float y) { return expr(x/y); } + + /// Output implementation. + /// \param out stream to write to + /// \param arg value to write + /// \return reference to stream + template static std::basic_ostream& write(std::basic_ostream &out, float arg) { return out << arg; } + + /// Input implementation. + /// \param in stream to read from + /// \param arg half to read into + /// \return reference to stream + template static std::basic_istream& read(std::basic_istream &in, half &arg) + { + float f; + if(in >> f) + arg = f; + return in; + } + + /// Modulo implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision division remainder stored in single-precision + static expr fmod(float x, float y) { return expr(std::fmod(x, y)); } + + /// Remainder implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision division remainder stored in single-precision + static expr remainder(float x, float y) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::remainder(x, y)); + #else + if(builtin_isnan(x) || builtin_isnan(y)) + return expr(std::numeric_limits::quiet_NaN()); + float ax = std::fabs(x), ay = std::fabs(y); + if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) + return expr(std::numeric_limits::quiet_NaN()); + if(ay >= 65536.0f) + return expr(x); + if(ax == ay) + return expr(builtin_signbit(x) ? -0.0f : 0.0f); + ax = std::fmod(ax, ay+ay); + float y2 = 0.5f * ay; + if(ax > y2) + { + ax -= ay; + if(ax >= y2) + ax -= ay; + } + return expr(builtin_signbit(x) ? -ax : ax); + #endif + } + + /// Remainder implementation. + /// \param x first operand + /// \param y second operand + /// \param quo address to store quotient bits at + /// \return Half-precision division remainder stored in single-precision + static expr remquo(float x, float y, int *quo) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::remquo(x, y, quo)); + #else + if(builtin_isnan(x) || builtin_isnan(y)) + return expr(std::numeric_limits::quiet_NaN()); + bool sign = builtin_signbit(x), qsign = static_cast(sign^builtin_signbit(y)); + float ax = std::fabs(x), ay = std::fabs(y); + if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) + return expr(std::numeric_limits::quiet_NaN()); + if(ay >= 65536.0f) + return expr(x); + if(ax == ay) + return *quo = qsign ? -1 : 1, expr(sign ? -0.0f : 0.0f); + ax = std::fmod(ax, 8.0f*ay); + int cquo = 0; + if(ax >= 4.0f * ay) + { + ax -= 4.0f * ay; + cquo += 4; + } + if(ax >= 2.0f * ay) + { + ax -= 2.0f * ay; + cquo += 2; + } + float y2 = 0.5f * ay; + if(ax > y2) + { + ax -= ay; + ++cquo; + if(ax >= y2) + { + ax -= ay; + ++cquo; + } + } + return *quo = qsign ? -cquo : cquo, expr(sign ? -ax : ax); + #endif + } + + /// Positive difference implementation. + /// \param x first operand + /// \param y second operand + /// \return Positive difference stored in single-precision + static expr fdim(float x, float y) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::fdim(x, y)); + #else + return expr((x<=y) ? 0.0f : (x-y)); + #endif + } + + /// Fused multiply-add implementation. + /// \param x first operand + /// \param y second operand + /// \param z third operand + /// \return \a x * \a y + \a z stored in single-precision + static expr fma(float x, float y, float z) + { + #if HALF_ENABLE_CPP11_CMATH && defined(FP_FAST_FMAF) + return expr(std::fma(x, y, z)); + #else + return expr(x*y+z); + #endif + } + + /// Get NaN. + /// \return Half-precision quiet NaN + static half nanh() { return half(binary, 0x7FFF); } + + /// Exponential implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr exp(float arg) { return expr(std::exp(arg)); } + + /// Exponential implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr expm1(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::expm1(arg)); + #else + return expr(static_cast(std::exp(static_cast(arg))-1.0)); + #endif + } + + /// Binary exponential implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr exp2(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::exp2(arg)); + #else + return expr(static_cast(std::exp(arg*0.69314718055994530941723212145818))); + #endif + } + + /// Logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log(float arg) { return expr(std::log(arg)); } + + /// Common logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log10(float arg) { return expr(std::log10(arg)); } + + /// Logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log1p(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::log1p(arg)); + #else + return expr(static_cast(std::log(1.0+arg))); + #endif + } + + /// Binary logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log2(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::log2(arg)); + #else + return expr(static_cast(std::log(static_cast(arg))*1.4426950408889634073599246810019)); + #endif + } + + /// Square root implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr sqrt(float arg) { return expr(std::sqrt(arg)); } + + /// Cubic root implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr cbrt(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::cbrt(arg)); + #else + if(builtin_isnan(arg) || builtin_isinf(arg)) + return expr(arg); + return expr(builtin_signbit(arg) ? -static_cast(std::pow(-static_cast(arg), 1.0/3.0)) : + static_cast(std::pow(static_cast(arg), 1.0/3.0))); + #endif + } + + /// Hypotenuse implementation. + /// \param x first argument + /// \param y second argument + /// \return function value stored in single-preicision + static expr hypot(float x, float y) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::hypot(x, y)); + #else + return expr((builtin_isinf(x) || builtin_isinf(y)) ? std::numeric_limits::infinity() : + static_cast(std::sqrt(static_cast(x)*x+static_cast(y)*y))); + #endif + } + + /// Power implementation. + /// \param base value to exponentiate + /// \param exp power to expontiate to + /// \return function value stored in single-preicision + static expr pow(float base, float exp) { return expr(std::pow(base, exp)); } + + /// Sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr sin(float arg) { return expr(std::sin(arg)); } + + /// Cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr cos(float arg) { return expr(std::cos(arg)); } + + /// Tan implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr tan(float arg) { return expr(std::tan(arg)); } + + /// Arc sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr asin(float arg) { return expr(std::asin(arg)); } + + /// Arc cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr acos(float arg) { return expr(std::acos(arg)); } + + /// Arc tangent implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr atan(float arg) { return expr(std::atan(arg)); } + + /// Arc tangent implementation. + /// \param x first argument + /// \param y second argument + /// \return function value stored in single-preicision + static expr atan2(float x, float y) { return expr(std::atan2(x, y)); } + + /// Hyperbolic sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr sinh(float arg) { return expr(std::sinh(arg)); } + + /// Hyperbolic cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr cosh(float arg) { return expr(std::cosh(arg)); } + + /// Hyperbolic tangent implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr tanh(float arg) { return expr(std::tanh(arg)); } + + /// Hyperbolic area sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr asinh(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::asinh(arg)); + #else + return expr((arg==-std::numeric_limits::infinity()) ? arg : static_cast(std::log(arg+std::sqrt(arg*arg+1.0)))); + #endif + } + + /// Hyperbolic area cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr acosh(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::acosh(arg)); + #else + return expr((arg<-1.0f) ? std::numeric_limits::quiet_NaN() : static_cast(std::log(arg+std::sqrt(arg*arg-1.0)))); + #endif + } + + /// Hyperbolic area tangent implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr atanh(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::atanh(arg)); + #else + return expr(static_cast(0.5*std::log((1.0+arg)/(1.0-arg)))); + #endif + } + + /// Error function implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr erf(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::erf(arg)); + #else + return expr(static_cast(erf(static_cast(arg)))); + #endif + } + + /// Complementary implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr erfc(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::erfc(arg)); + #else + return expr(static_cast(1.0-erf(static_cast(arg)))); + #endif + } + + /// Gamma logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr lgamma(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::lgamma(arg)); + #else + if(builtin_isinf(arg)) + return expr(std::numeric_limits::infinity()); + if(arg < 0.0f) + { + float i, f = std::modf(-arg, &i); + if(f == 0.0f) + return expr(std::numeric_limits::infinity()); + return expr(static_cast(1.1447298858494001741434273513531- + std::log(std::abs(std::sin(3.1415926535897932384626433832795*f)))-lgamma(1.0-arg))); + } + return expr(static_cast(lgamma(static_cast(arg)))); + #endif + } + + /// Gamma implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr tgamma(float arg) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::tgamma(arg)); + #else + if(arg == 0.0f) + return builtin_signbit(arg) ? expr(-std::numeric_limits::infinity()) : expr(std::numeric_limits::infinity()); + if(arg < 0.0f) + { + float i, f = std::modf(-arg, &i); + if(f == 0.0f) + return expr(std::numeric_limits::quiet_NaN()); + double value = 3.1415926535897932384626433832795 / (std::sin(3.1415926535897932384626433832795*f)*std::exp(lgamma(1.0-arg))); + return expr(static_cast((std::fmod(i, 2.0f)==0.0f) ? -value : value)); + } + if(builtin_isinf(arg)) + return expr(arg); + return expr(static_cast(std::exp(lgamma(static_cast(arg))))); + #endif + } + + /// Floor implementation. + /// \param arg value to round + /// \return rounded value + static half floor(half arg) { return half(binary, round_half(arg.data_)); } + + /// Ceiling implementation. + /// \param arg value to round + /// \return rounded value + static half ceil(half arg) { return half(binary, round_half(arg.data_)); } + + /// Truncation implementation. + /// \param arg value to round + /// \return rounded value + static half trunc(half arg) { return half(binary, round_half(arg.data_)); } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static half round(half arg) { return half(binary, round_half_up(arg.data_)); } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long lround(half arg) { return detail::half2int_up(arg.data_); } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static half rint(half arg) { return half(binary, round_half(arg.data_)); } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long lrint(half arg) { return detail::half2int(arg.data_); } + + #if HALF_ENABLE_CPP11_LONG_LONG + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long long llround(half arg) { return detail::half2int_up(arg.data_); } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long long llrint(half arg) { return detail::half2int(arg.data_); } + #endif + + /// Decompression implementation. + /// \param arg number to decompress + /// \param exp address to store exponent at + /// \return normalized significant + static half frexp(half arg, int *exp) + { + int m = arg.data_ & 0x7FFF, e = -14; + if(m >= 0x7C00 || !m) + return *exp = 0, arg; + for(; m<0x400; m<<=1,--e) ; + return *exp = e+(m>>10), half(binary, (arg.data_&0x8000)|0x3800|(m&0x3FF)); + } + + /// Decompression implementation. + /// \param arg number to decompress + /// \param iptr address to store integer part at + /// \return fractional part + static half modf(half arg, half *iptr) + { + unsigned int e = arg.data_ & 0x7FFF; + if(e >= 0x6400) + return *iptr = arg, half(binary, arg.data_&(0x8000U|-(e>0x7C00))); + if(e < 0x3C00) + return iptr->data_ = arg.data_ & 0x8000, arg; + e >>= 10; + unsigned int mask = (1<<(25-e)) - 1, m = arg.data_ & mask; + iptr->data_ = arg.data_ & ~mask; + if(!m) + return half(binary, arg.data_&0x8000); + for(; m<0x400; m<<=1,--e) ; + return half(binary, static_cast((arg.data_&0x8000)|(e<<10)|(m&0x3FF))); + } + + /// Scaling implementation. + /// \param arg number to scale + /// \param exp power of two to scale by + /// \return scaled number + static half scalbln(half arg, long exp) + { + unsigned int m = arg.data_ & 0x7FFF; + if(m >= 0x7C00 || !m) + return arg; + for(; m<0x400; m<<=1,--exp) ; + exp += m >> 10; + uint16 value = arg.data_ & 0x8000; + if(exp > 30) + { + if(half::round_style == std::round_toward_zero) + value |= 0x7BFF; + else if(half::round_style == std::round_toward_infinity) + value |= 0x7C00 - (value>>15); + else if(half::round_style == std::round_toward_neg_infinity) + value |= 0x7BFF + (value>>15); + else + value |= 0x7C00; + } + else if(exp > 0) + value |= (exp<<10) | (m&0x3FF); + else if(exp > -11) + { + m = (m&0x3FF) | 0x400; + if(half::round_style == std::round_to_nearest) + { + m += 1 << -exp; + #if HALF_ROUND_TIES_TO_EVEN + m -= (m>>(1-exp)) & 1; + #endif + } + else if(half::round_style == std::round_toward_infinity) + m += ((value>>15)-1) & ((1<<(1-exp))-1U); + else if(half::round_style == std::round_toward_neg_infinity) + m += -(value>>15) & ((1<<(1-exp))-1U); + value |= m >> (1-exp); + } + else if(half::round_style == std::round_toward_infinity) + value -= (value>>15) - 1; + else if(half::round_style == std::round_toward_neg_infinity) + value += value >> 15; + return half(binary, value); + } + + /// Exponent implementation. + /// \param arg number to query + /// \return floating point exponent + static int ilogb(half arg) + { + int abs = arg.data_ & 0x7FFF; + if(!abs) + return FP_ILOGB0; + if(abs < 0x7C00) + { + int exp = (abs>>10) - 15; + if(abs < 0x400) + for(; abs<0x200; abs<<=1,--exp) ; + return exp; + } + if(abs > 0x7C00) + return FP_ILOGBNAN; + return INT_MAX; + } + + /// Exponent implementation. + /// \param arg number to query + /// \return floating point exponent + static half logb(half arg) + { + int abs = arg.data_ & 0x7FFF; + if(!abs) + return half(binary, 0xFC00); + if(abs < 0x7C00) + { + int exp = (abs>>10) - 15; + if(abs < 0x400) + for(; abs<0x200; abs<<=1,--exp) ; + uint16 bits = (exp<0) << 15; + if(exp) + { + unsigned int m = std::abs(exp) << 6, e = 18; + for(; m<0x400; m<<=1,--e) ; + bits |= (e<<10) + m; + } + return half(binary, bits); + } + if(abs > 0x7C00) + return arg; + return half(binary, 0x7C00); + } + + /// Enumeration implementation. + /// \param from number to increase/decrease + /// \param to direction to enumerate into + /// \return next representable number + static half nextafter(half from, half to) + { + uint16 fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF; + if(fabs > 0x7C00) + return from; + if(tabs > 0x7C00 || from.data_ == to.data_ || !(fabs|tabs)) + return to; + if(!fabs) + return half(binary, (to.data_&0x8000)+1); + bool lt = ((fabs==from.data_) ? static_cast(fabs) : -static_cast(fabs)) < + ((tabs==to.data_) ? static_cast(tabs) : -static_cast(tabs)); + return half(binary, from.data_+(((from.data_>>15)^static_cast(lt))<<1)-1); + } + + /// Enumeration implementation. + /// \param from number to increase/decrease + /// \param to direction to enumerate into + /// \return next representable number + static half nexttoward(half from, long double to) + { + if(isnan(from)) + return from; + long double lfrom = static_cast(from); + if(builtin_isnan(to) || lfrom == to) + return half(static_cast(to)); + if(!(from.data_&0x7FFF)) + return half(binary, (static_cast(builtin_signbit(to))<<15)+1); + return half(binary, from.data_+(((from.data_>>15)^static_cast(lfrom0x3FF) ? ((abs>=0x7C00) ? ((abs>0x7C00) ? FP_NAN : FP_INFINITE) : FP_NORMAL) :FP_SUBNORMAL) : FP_ZERO; + } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if finite number + /// \retval false else + static bool isfinite(half arg) { return (arg.data_&0x7C00) != 0x7C00; } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if infinite number + /// \retval false else + static bool isinf(half arg) { return (arg.data_&0x7FFF) == 0x7C00; } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if not a number + /// \retval false else + static bool isnan(half arg) { return (arg.data_&0x7FFF) > 0x7C00; } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if normal number + /// \retval false else + static bool isnormal(half arg) { return ((arg.data_&0x7C00)!=0) & ((arg.data_&0x7C00)!=0x7C00); } + + /// Sign bit implementation. + /// \param arg value to check + /// \retval true if signed + /// \retval false if unsigned + static bool signbit(half arg) { return (arg.data_&0x8000) != 0; } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if operands equal + /// \retval false else + static bool isequal(half x, half y) { return (x.data_==y.data_ || !((x.data_|y.data_)&0x7FFF)) && !isnan(x); } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if operands not equal + /// \retval false else + static bool isnotequal(half x, half y) { return (x.data_!=y.data_ && ((x.data_|y.data_)&0x7FFF)) || isnan(x); } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x > \a y + /// \retval false else + static bool isgreater(half x, half y) + { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs)); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x >= \a y + /// \retval false else + static bool isgreaterequal(half x, half y) + { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) >= ((yabs==y.data_) ? yabs : -yabs)); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x < \a y + /// \retval false else + static bool isless(half x, half y) + { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs)); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x <= \a y + /// \retval false else + static bool islessequal(half x, half y) + { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) <= ((yabs==y.data_) ? yabs : -yabs)); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if either \a x > \a y nor \a x < \a y + /// \retval false else + static bool islessgreater(half x, half y) + { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + if(xabs > 0x7C00 || yabs > 0x7C00) + return false; + int a = (xabs==x.data_) ? xabs : -xabs, b = (yabs==y.data_) ? yabs : -yabs; + return a < b || a > b; + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if operand unordered + /// \retval false else + static bool isunordered(half x, half y) { return isnan(x) || isnan(y); } + + private: + static double erf(double arg) + { + if(builtin_isinf(arg)) + return (arg<0.0) ? -1.0 : 1.0; + double x2 = arg * arg, ax2 = 0.147 * x2, value = std::sqrt(1.0-std::exp(-x2*(1.2732395447351626861510701069801+ax2)/(1.0+ax2))); + return builtin_signbit(arg) ? -value : value; + } + + static double lgamma(double arg) + { + double v = 1.0; + for(; arg<8.0; ++arg) v *= arg; + double w = 1.0 / (arg*arg); + return (((((((-0.02955065359477124183006535947712*w+0.00641025641025641025641025641026)*w+ + -0.00191752691752691752691752691753)*w+8.4175084175084175084175084175084e-4)*w+ + -5.952380952380952380952380952381e-4)*w+7.9365079365079365079365079365079e-4)*w+ + -0.00277777777777777777777777777778)*w+0.08333333333333333333333333333333)/arg + + 0.91893853320467274178032973640562 - std::log(v) - arg + (arg-0.5) * std::log(arg); + } + }; + + /// Wrapper for unary half-precision functions needing specialization for individual argument types. + /// \tparam T argument type + template struct unary_specialized + { + /// Negation implementation. + /// \param arg value to negate + /// \return negated value + static HALF_CONSTEXPR half negate(half arg) { return half(binary, arg.data_^0x8000); } + + /// Absolute value implementation. + /// \param arg function argument + /// \return absolute value + static half fabs(half arg) { return half(binary, arg.data_&0x7FFF); } + }; + template<> struct unary_specialized + { + static HALF_CONSTEXPR expr negate(float arg) { return expr(-arg); } + static expr fabs(float arg) { return expr(std::fabs(arg)); } + }; + + /// Wrapper for binary half-precision functions needing specialization for individual argument types. + /// \tparam T first argument type + /// \tparam U first argument type + template struct binary_specialized + { + /// Minimum implementation. + /// \param x first operand + /// \param y second operand + /// \return minimum value + static expr fmin(float x, float y) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::fmin(x, y)); + #else + if(builtin_isnan(x)) + return expr(y); + if(builtin_isnan(y)) + return expr(x); + return expr(std::min(x, y)); + #endif + } + + /// Maximum implementation. + /// \param x first operand + /// \param y second operand + /// \return maximum value + static expr fmax(float x, float y) + { + #if HALF_ENABLE_CPP11_CMATH + return expr(std::fmax(x, y)); + #else + if(builtin_isnan(x)) + return expr(y); + if(builtin_isnan(y)) + return expr(x); + return expr(std::max(x, y)); + #endif + } + }; + template<> struct binary_specialized + { + static half fmin(half x, half y) + { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + if(xabs > 0x7C00) + return y; + if(yabs > 0x7C00) + return x; + return (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs)) ? y : x; + } + static half fmax(half x, half y) + { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + if(xabs > 0x7C00) + return y; + if(yabs > 0x7C00) + return x; + return (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs)) ? y : x; + } + }; + + /// Helper class for half casts. + /// This class template has to be specialized for all valid cast argument to define an appropriate static `cast` member + /// function and a corresponding `type` member denoting its return type. + /// \tparam T destination type + /// \tparam U source type + /// \tparam R rounding mode to use + template struct half_caster {}; + template struct half_caster + { + #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, "half_cast from non-arithmetic type unsupported"); + #endif + + static half cast(U arg) { return cast_impl(arg, is_float()); }; + + private: + static half cast_impl(U arg, true_type) { return half(binary, float2half(arg)); } + static half cast_impl(U arg, false_type) { return half(binary, int2half(arg)); } + }; + template struct half_caster + { + #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, "half_cast to non-arithmetic type unsupported"); + #endif + + static T cast(half arg) { return cast_impl(arg, is_float()); } + + private: + static T cast_impl(half arg, true_type) { return half2float(arg.data_); } + static T cast_impl(half arg, false_type) { return half2int(arg.data_); } + }; + template struct half_caster + { + #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, "half_cast to non-arithmetic type unsupported"); + #endif + + static T cast(expr arg) { return cast_impl(arg, is_float()); } + + private: + static T cast_impl(float arg, true_type) { return static_cast(arg); } + static T cast_impl(half arg, false_type) { return half2int(arg.data_); } + }; + template struct half_caster + { + static half cast(half arg) { return arg; } + }; + template struct half_caster : half_caster {}; + + /// \name Comparison operators + /// \{ + + /// Comparison for equality. + /// \param x first operand + /// \param y second operand + /// \retval true if operands equal + /// \retval false else + template typename enable::type operator==(T x, U y) { return functions::isequal(x, y); } + + /// Comparison for inequality. + /// \param x first operand + /// \param y second operand + /// \retval true if operands not equal + /// \retval false else + template typename enable::type operator!=(T x, U y) { return functions::isnotequal(x, y); } + + /// Comparison for less than. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less than \a y + /// \retval false else + template typename enable::type operator<(T x, U y) { return functions::isless(x, y); } + + /// Comparison for greater than. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater than \a y + /// \retval false else + template typename enable::type operator>(T x, U y) { return functions::isgreater(x, y); } + + /// Comparison for less equal. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less equal \a y + /// \retval false else + template typename enable::type operator<=(T x, U y) { return functions::islessequal(x, y); } + + /// Comparison for greater equal. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater equal \a y + /// \retval false else + template typename enable::type operator>=(T x, U y) { return functions::isgreaterequal(x, y); } + + /// \} + /// \name Arithmetic operators + /// \{ + + /// Add halfs. + /// \param x left operand + /// \param y right operand + /// \return sum of half expressions + template typename enable::type operator+(T x, U y) { return functions::plus(x, y); } + + /// Subtract halfs. + /// \param x left operand + /// \param y right operand + /// \return difference of half expressions + template typename enable::type operator-(T x, U y) { return functions::minus(x, y); } + + /// Multiply halfs. + /// \param x left operand + /// \param y right operand + /// \return product of half expressions + template typename enable::type operator*(T x, U y) { return functions::multiplies(x, y); } + + /// Divide halfs. + /// \param x left operand + /// \param y right operand + /// \return quotient of half expressions + template typename enable::type operator/(T x, U y) { return functions::divides(x, y); } + + /// Identity. + /// \param arg operand + /// \return uncahnged operand + template HALF_CONSTEXPR typename enable::type operator+(T arg) { return arg; } + + /// Negation. + /// \param arg operand + /// \return negated operand + template HALF_CONSTEXPR typename enable::type operator-(T arg) { return unary_specialized::negate(arg); } + + /// \} + /// \name Input and output + /// \{ + + /// Output operator. + /// \param out output stream to write into + /// \param arg half expression to write + /// \return reference to output stream + template typename enable&,T>::type + operator<<(std::basic_ostream &out, T arg) { return functions::write(out, arg); } + + /// Input operator. + /// \param in input stream to read from + /// \param arg half to read into + /// \return reference to input stream + template std::basic_istream& + operator>>(std::basic_istream &in, half &arg) { return functions::read(in, arg); } + + /// \} + /// \name Basic mathematical operations + /// \{ + + /// Absolute value. + /// \param arg operand + /// \return absolute value of \a arg +// template typename enable::type abs(T arg) { return unary_specialized::fabs(arg); } + inline half abs(half arg) { return unary_specialized::fabs(arg); } + inline expr abs(expr arg) { return unary_specialized::fabs(arg); } + + /// Absolute value. + /// \param arg operand + /// \return absolute value of \a arg +// template typename enable::type fabs(T arg) { return unary_specialized::fabs(arg); } + inline half fabs(half arg) { return unary_specialized::fabs(arg); } + inline expr fabs(expr arg) { return unary_specialized::fabs(arg); } + + /// Remainder of division. + /// \param x first operand + /// \param y second operand + /// \return remainder of floating point division. +// template typename enable::type fmod(T x, U y) { return functions::fmod(x, y); } + inline expr fmod(half x, half y) { return functions::fmod(x, y); } + inline expr fmod(half x, expr y) { return functions::fmod(x, y); } + inline expr fmod(expr x, half y) { return functions::fmod(x, y); } + inline expr fmod(expr x, expr y) { return functions::fmod(x, y); } + + /// Remainder of division. + /// \param x first operand + /// \param y second operand + /// \return remainder of floating point division. +// template typename enable::type remainder(T x, U y) { return functions::remainder(x, y); } + inline expr remainder(half x, half y) { return functions::remainder(x, y); } + inline expr remainder(half x, expr y) { return functions::remainder(x, y); } + inline expr remainder(expr x, half y) { return functions::remainder(x, y); } + inline expr remainder(expr x, expr y) { return functions::remainder(x, y); } + + /// Remainder of division. + /// \param x first operand + /// \param y second operand + /// \param quo address to store some bits of quotient at + /// \return remainder of floating point division. +// template typename enable::type remquo(T x, U y, int *quo) { return functions::remquo(x, y, quo); } + inline expr remquo(half x, half y, int *quo) { return functions::remquo(x, y, quo); } + inline expr remquo(half x, expr y, int *quo) { return functions::remquo(x, y, quo); } + inline expr remquo(expr x, half y, int *quo) { return functions::remquo(x, y, quo); } + inline expr remquo(expr x, expr y, int *quo) { return functions::remquo(x, y, quo); } + + /// Fused multiply add. + /// \param x first operand + /// \param y second operand + /// \param z third operand + /// \return ( \a x * \a y ) + \a z rounded as one operation. +// template typename enable::type fma(T x, U y, V z) { return functions::fma(x, y, z); } + inline expr fma(half x, half y, half z) { return functions::fma(x, y, z); } + inline expr fma(half x, half y, expr z) { return functions::fma(x, y, z); } + inline expr fma(half x, expr y, half z) { return functions::fma(x, y, z); } + inline expr fma(half x, expr y, expr z) { return functions::fma(x, y, z); } + inline expr fma(expr x, half y, half z) { return functions::fma(x, y, z); } + inline expr fma(expr x, half y, expr z) { return functions::fma(x, y, z); } + inline expr fma(expr x, expr y, half z) { return functions::fma(x, y, z); } + inline expr fma(expr x, expr y, expr z) { return functions::fma(x, y, z); } + + /// Maximum of half expressions. + /// \param x first operand + /// \param y second operand + /// \return maximum of operands +// template typename result::type fmax(T x, U y) { return binary_specialized::fmax(x, y); } + inline half fmax(half x, half y) { return binary_specialized::fmax(x, y); } + inline expr fmax(half x, expr y) { return binary_specialized::fmax(x, y); } + inline expr fmax(expr x, half y) { return binary_specialized::fmax(x, y); } + inline expr fmax(expr x, expr y) { return binary_specialized::fmax(x, y); } + + /// Minimum of half expressions. + /// \param x first operand + /// \param y second operand + /// \return minimum of operands +// template typename result::type fmin(T x, U y) { return binary_specialized::fmin(x, y); } + inline half fmin(half x, half y) { return binary_specialized::fmin(x, y); } + inline expr fmin(half x, expr y) { return binary_specialized::fmin(x, y); } + inline expr fmin(expr x, half y) { return binary_specialized::fmin(x, y); } + inline expr fmin(expr x, expr y) { return binary_specialized::fmin(x, y); } + + /// Positive difference. + /// \param x first operand + /// \param y second operand + /// \return \a x - \a y or 0 if difference negative +// template typename enable::type fdim(T x, U y) { return functions::fdim(x, y); } + inline expr fdim(half x, half y) { return functions::fdim(x, y); } + inline expr fdim(half x, expr y) { return functions::fdim(x, y); } + inline expr fdim(expr x, half y) { return functions::fdim(x, y); } + inline expr fdim(expr x, expr y) { return functions::fdim(x, y); } + + /// Get NaN value. + /// \return quiet NaN + inline half nanh(const char*) { return functions::nanh(); } + + /// \} + /// \name Exponential functions + /// \{ + + /// Exponential function. + /// \param arg function argument + /// \return e raised to \a arg +// template typename enable::type exp(T arg) { return functions::exp(arg); } + inline expr exp(half arg) { return functions::exp(arg); } + inline expr exp(expr arg) { return functions::exp(arg); } + + /// Exponential minus one. + /// \param arg function argument + /// \return e raised to \a arg subtracted by 1 +// template typename enable::type expm1(T arg) { return functions::expm1(arg); } + inline expr expm1(half arg) { return functions::expm1(arg); } + inline expr expm1(expr arg) { return functions::expm1(arg); } + + /// Binary exponential. + /// \param arg function argument + /// \return 2 raised to \a arg +// template typename enable::type exp2(T arg) { return functions::exp2(arg); } + inline expr exp2(half arg) { return functions::exp2(arg); } + inline expr exp2(expr arg) { return functions::exp2(arg); } + + /// Natural logorithm. + /// \param arg function argument + /// \return logarithm of \a arg to base e +// template typename enable::type log(T arg) { return functions::log(arg); } + inline expr log(half arg) { return functions::log(arg); } + inline expr log(expr arg) { return functions::log(arg); } + + /// Common logorithm. + /// \param arg function argument + /// \return logarithm of \a arg to base 10 +// template typename enable::type log10(T arg) { return functions::log10(arg); } + inline expr log10(half arg) { return functions::log10(arg); } + inline expr log10(expr arg) { return functions::log10(arg); } + + /// Natural logorithm. + /// \param arg function argument + /// \return logarithm of \a arg plus 1 to base e +// template typename enable::type log1p(T arg) { return functions::log1p(arg); } + inline expr log1p(half arg) { return functions::log1p(arg); } + inline expr log1p(expr arg) { return functions::log1p(arg); } + + /// Binary logorithm. + /// \param arg function argument + /// \return logarithm of \a arg to base 2 +// template typename enable::type log2(T arg) { return functions::log2(arg); } + inline expr log2(half arg) { return functions::log2(arg); } + inline expr log2(expr arg) { return functions::log2(arg); } + + /// \} + /// \name Power functions + /// \{ + + /// Square root. + /// \param arg function argument + /// \return square root of \a arg +// template typename enable::type sqrt(T arg) { return functions::sqrt(arg); } + inline expr sqrt(half arg) { return functions::sqrt(arg); } + inline expr sqrt(expr arg) { return functions::sqrt(arg); } + + /// Cubic root. + /// \param arg function argument + /// \return cubic root of \a arg +// template typename enable::type cbrt(T arg) { return functions::cbrt(arg); } + inline expr cbrt(half arg) { return functions::cbrt(arg); } + inline expr cbrt(expr arg) { return functions::cbrt(arg); } + + /// Hypotenuse function. + /// \param x first argument + /// \param y second argument + /// \return square root of sum of squares without internal over- or underflows +// template typename enable::type hypot(T x, U y) { return functions::hypot(x, y); } + inline expr hypot(half x, half y) { return functions::hypot(x, y); } + inline expr hypot(half x, expr y) { return functions::hypot(x, y); } + inline expr hypot(expr x, half y) { return functions::hypot(x, y); } + inline expr hypot(expr x, expr y) { return functions::hypot(x, y); } + + /// Power function. + /// \param base first argument + /// \param exp second argument + /// \return \a base raised to \a exp +// template typename enable::type pow(T base, U exp) { return functions::pow(base, exp); } + inline expr pow(half base, half exp) { return functions::pow(base, exp); } + inline expr pow(half base, expr exp) { return functions::pow(base, exp); } + inline expr pow(expr base, half exp) { return functions::pow(base, exp); } + inline expr pow(expr base, expr exp) { return functions::pow(base, exp); } + + /// \} + /// \name Trigonometric functions + /// \{ + + /// Sine function. + /// \param arg function argument + /// \return sine value of \a arg +// template typename enable::type sin(T arg) { return functions::sin(arg); } + inline expr sin(half arg) { return functions::sin(arg); } + inline expr sin(expr arg) { return functions::sin(arg); } + + /// Cosine function. + /// \param arg function argument + /// \return cosine value of \a arg +// template typename enable::type cos(T arg) { return functions::cos(arg); } + inline expr cos(half arg) { return functions::cos(arg); } + inline expr cos(expr arg) { return functions::cos(arg); } + + /// Tangent function. + /// \param arg function argument + /// \return tangent value of \a arg +// template typename enable::type tan(T arg) { return functions::tan(arg); } + inline expr tan(half arg) { return functions::tan(arg); } + inline expr tan(expr arg) { return functions::tan(arg); } + + /// Arc sine. + /// \param arg function argument + /// \return arc sine value of \a arg +// template typename enable::type asin(T arg) { return functions::asin(arg); } + inline expr asin(half arg) { return functions::asin(arg); } + inline expr asin(expr arg) { return functions::asin(arg); } + + /// Arc cosine function. + /// \param arg function argument + /// \return arc cosine value of \a arg +// template typename enable::type acos(T arg) { return functions::acos(arg); } + inline expr acos(half arg) { return functions::acos(arg); } + inline expr acos(expr arg) { return functions::acos(arg); } + + /// Arc tangent function. + /// \param arg function argument + /// \return arc tangent value of \a arg +// template typename enable::type atan(T arg) { return functions::atan(arg); } + inline expr atan(half arg) { return functions::atan(arg); } + inline expr atan(expr arg) { return functions::atan(arg); } + + /// Arc tangent function. + /// \param x first argument + /// \param y second argument + /// \return arc tangent value +// template typename enable::type atan2(T x, U y) { return functions::atan2(x, y); } + inline expr atan2(half x, half y) { return functions::atan2(x, y); } + inline expr atan2(half x, expr y) { return functions::atan2(x, y); } + inline expr atan2(expr x, half y) { return functions::atan2(x, y); } + inline expr atan2(expr x, expr y) { return functions::atan2(x, y); } + + /// \} + /// \name Hyperbolic functions + /// \{ + + /// Hyperbolic sine. + /// \param arg function argument + /// \return hyperbolic sine value of \a arg +// template typename enable::type sinh(T arg) { return functions::sinh(arg); } + inline expr sinh(half arg) { return functions::sinh(arg); } + inline expr sinh(expr arg) { return functions::sinh(arg); } + + /// Hyperbolic cosine. + /// \param arg function argument + /// \return hyperbolic cosine value of \a arg +// template typename enable::type cosh(T arg) { return functions::cosh(arg); } + inline expr cosh(half arg) { return functions::cosh(arg); } + inline expr cosh(expr arg) { return functions::cosh(arg); } + + /// Hyperbolic tangent. + /// \param arg function argument + /// \return hyperbolic tangent value of \a arg +// template typename enable::type tanh(T arg) { return functions::tanh(arg); } + inline expr tanh(half arg) { return functions::tanh(arg); } + inline expr tanh(expr arg) { return functions::tanh(arg); } + + /// Hyperbolic area sine. + /// \param arg function argument + /// \return area sine value of \a arg +// template typename enable::type asinh(T arg) { return functions::asinh(arg); } + inline expr asinh(half arg) { return functions::asinh(arg); } + inline expr asinh(expr arg) { return functions::asinh(arg); } + + /// Hyperbolic area cosine. + /// \param arg function argument + /// \return area cosine value of \a arg +// template typename enable::type acosh(T arg) { return functions::acosh(arg); } + inline expr acosh(half arg) { return functions::acosh(arg); } + inline expr acosh(expr arg) { return functions::acosh(arg); } + + /// Hyperbolic area tangent. + /// \param arg function argument + /// \return area tangent value of \a arg +// template typename enable::type atanh(T arg) { return functions::atanh(arg); } + inline expr atanh(half arg) { return functions::atanh(arg); } + inline expr atanh(expr arg) { return functions::atanh(arg); } + + /// \} + /// \name Error and gamma functions + /// \{ + + /// Error function. + /// \param arg function argument + /// \return error function value of \a arg +// template typename enable::type erf(T arg) { return functions::erf(arg); } + inline expr erf(half arg) { return functions::erf(arg); } + inline expr erf(expr arg) { return functions::erf(arg); } + + /// Complementary error function. + /// \param arg function argument + /// \return 1 minus error function value of \a arg +// template typename enable::type erfc(T arg) { return functions::erfc(arg); } + inline expr erfc(half arg) { return functions::erfc(arg); } + inline expr erfc(expr arg) { return functions::erfc(arg); } + + /// Natural logarithm of gamma function. + /// \param arg function argument + /// \return natural logarith of gamma function for \a arg +// template typename enable::type lgamma(T arg) { return functions::lgamma(arg); } + inline expr lgamma(half arg) { return functions::lgamma(arg); } + inline expr lgamma(expr arg) { return functions::lgamma(arg); } + + /// Gamma function. + /// \param arg function argument + /// \return gamma function value of \a arg +// template typename enable::type tgamma(T arg) { return functions::tgamma(arg); } + inline expr tgamma(half arg) { return functions::tgamma(arg); } + inline expr tgamma(expr arg) { return functions::tgamma(arg); } + + /// \} + /// \name Rounding + /// \{ + + /// Nearest integer not less than half value. + /// \param arg half to round + /// \return nearest integer not less than \a arg +// template typename enable::type ceil(T arg) { return functions::ceil(arg); } + inline half ceil(half arg) { return functions::ceil(arg); } + inline half ceil(expr arg) { return functions::ceil(arg); } + + /// Nearest integer not greater than half value. + /// \param arg half to round + /// \return nearest integer not greater than \a arg +// template typename enable::type floor(T arg) { return functions::floor(arg); } + inline half floor(half arg) { return functions::floor(arg); } + inline half floor(expr arg) { return functions::floor(arg); } + + /// Nearest integer not greater in magnitude than half value. + /// \param arg half to round + /// \return nearest integer not greater in magnitude than \a arg +// template typename enable::type trunc(T arg) { return functions::trunc(arg); } + inline half trunc(half arg) { return functions::trunc(arg); } + inline half trunc(expr arg) { return functions::trunc(arg); } + + /// Nearest integer. + /// \param arg half to round + /// \return nearest integer, rounded away from zero in half-way cases +// template typename enable::type round(T arg) { return functions::round(arg); } + inline half round(half arg) { return functions::round(arg); } + inline half round(expr arg) { return functions::round(arg); } + + /// Nearest integer. + /// \param arg half to round + /// \return nearest integer, rounded away from zero in half-way cases +// template typename enable::type lround(T arg) { return functions::lround(arg); } + inline long lround(half arg) { return functions::lround(arg); } + inline long lround(expr arg) { return functions::lround(arg); } + + /// Nearest integer using half's internal rounding mode. + /// \param arg half expression to round + /// \return nearest integer using default rounding mode +// template typename enable::type nearbyint(T arg) { return functions::nearbyint(arg); } + inline half nearbyint(half arg) { return functions::rint(arg); } + inline half nearbyint(expr arg) { return functions::rint(arg); } + + /// Nearest integer using half's internal rounding mode. + /// \param arg half expression to round + /// \return nearest integer using default rounding mode +// template typename enable::type rint(T arg) { return functions::rint(arg); } + inline half rint(half arg) { return functions::rint(arg); } + inline half rint(expr arg) { return functions::rint(arg); } + + /// Nearest integer using half's internal rounding mode. + /// \param arg half expression to round + /// \return nearest integer using default rounding mode +// template typename enable::type lrint(T arg) { return functions::lrint(arg); } + inline long lrint(half arg) { return functions::lrint(arg); } + inline long lrint(expr arg) { return functions::lrint(arg); } + #if HALF_ENABLE_CPP11_LONG_LONG + /// Nearest integer. + /// \param arg half to round + /// \return nearest integer, rounded away from zero in half-way cases +// template typename enable::type llround(T arg) { return functions::llround(arg); } + inline long long llround(half arg) { return functions::llround(arg); } + inline long long llround(expr arg) { return functions::llround(arg); } + + /// Nearest integer using half's internal rounding mode. + /// \param arg half expression to round + /// \return nearest integer using default rounding mode +// template typename enable::type llrint(T arg) { return functions::llrint(arg); } + inline long long llrint(half arg) { return functions::llrint(arg); } + inline long long llrint(expr arg) { return functions::llrint(arg); } + #endif + + /// \} + /// \name Floating point manipulation + /// \{ + + /// Decompress floating point number. + /// \param arg number to decompress + /// \param exp address to store exponent at + /// \return significant in range [0.5, 1) +// template typename enable::type frexp(T arg, int *exp) { return functions::frexp(arg, exp); } + inline half frexp(half arg, int *exp) { return functions::frexp(arg, exp); } + inline half frexp(expr arg, int *exp) { return functions::frexp(arg, exp); } + + /// Multiply by power of two. + /// \param arg number to modify + /// \param exp power of two to multiply with + /// \return \a arg multplied by 2 raised to \a exp +// template typename enable::type ldexp(T arg, int exp) { return functions::scalbln(arg, exp); } + inline half ldexp(half arg, int exp) { return functions::scalbln(arg, exp); } + inline half ldexp(expr arg, int exp) { return functions::scalbln(arg, exp); } + + /// Extract integer and fractional parts. + /// \param arg number to decompress + /// \param iptr address to store integer part at + /// \return fractional part +// template typename enable::type modf(T arg, half *iptr) { return functions::modf(arg, iptr); } + inline half modf(half arg, half *iptr) { return functions::modf(arg, iptr); } + inline half modf(expr arg, half *iptr) { return functions::modf(arg, iptr); } + + /// Multiply by power of two. + /// \param arg number to modify + /// \param exp power of two to multiply with + /// \return \a arg multplied by 2 raised to \a exp +// template typename enable::type scalbn(T arg, int exp) { return functions::scalbln(arg, exp); } + inline half scalbn(half arg, int exp) { return functions::scalbln(arg, exp); } + inline half scalbn(expr arg, int exp) { return functions::scalbln(arg, exp); } + + /// Multiply by power of two. + /// \param arg number to modify + /// \param exp power of two to multiply with + /// \return \a arg multplied by 2 raised to \a exp +// template typename enable::type scalbln(T arg, long exp) { return functions::scalbln(arg, exp); } + inline half scalbln(half arg, long exp) { return functions::scalbln(arg, exp); } + inline half scalbln(expr arg, long exp) { return functions::scalbln(arg, exp); } + + /// Extract exponent. + /// \param arg number to query + /// \return floating point exponent + /// \retval FP_ILOGB0 for zero + /// \retval FP_ILOGBNAN for NaN + /// \retval MAX_INT for infinity +// template typename enable::type ilogb(T arg) { return functions::ilogb(arg); } + inline int ilogb(half arg) { return functions::ilogb(arg); } + inline int ilogb(expr arg) { return functions::ilogb(arg); } + + /// Extract exponent. + /// \param arg number to query + /// \return floating point exponent +// template typename enable::type logb(T arg) { return functions::logb(arg); } + inline half logb(half arg) { return functions::logb(arg); } + inline half logb(expr arg) { return functions::logb(arg); } + + /// Next representable value. + /// \param from value to compute next representable value for + /// \param to direction towards which to compute next value + /// \return next representable value after \a from in direction towards \a to +// template typename enable::type nextafter(T from, U to) { return functions::nextafter(from, to); } + inline half nextafter(half from, half to) { return functions::nextafter(from, to); } + inline half nextafter(half from, expr to) { return functions::nextafter(from, to); } + inline half nextafter(expr from, half to) { return functions::nextafter(from, to); } + inline half nextafter(expr from, expr to) { return functions::nextafter(from, to); } + + /// Next representable value. + /// \param from value to compute next representable value for + /// \param to direction towards which to compute next value + /// \return next representable value after \a from in direction towards \a to +// template typename enable::type nexttoward(T from, long double to) { return functions::nexttoward(from, to); } + inline half nexttoward(half from, long double to) { return functions::nexttoward(from, to); } + inline half nexttoward(expr from, long double to) { return functions::nexttoward(from, to); } + + /// Take sign. + /// \param x value to change sign for + /// \param y value to take sign from + /// \return value equal to \a x in magnitude and to \a y in sign +// template typename enable::type copysign(T x, U y) { return functions::copysign(x, y); } + inline half copysign(half x, half y) { return functions::copysign(x, y); } + inline half copysign(half x, expr y) { return functions::copysign(x, y); } + inline half copysign(expr x, half y) { return functions::copysign(x, y); } + inline half copysign(expr x, expr y) { return functions::copysign(x, y); } + + /// \} + /// \name Floating point classification + /// \{ + + + /// Classify floating point value. + /// \param arg number to classify + /// \retval FP_ZERO for positive and negative zero + /// \retval FP_SUBNORMAL for subnormal numbers + /// \retval FP_INFINITY for positive and negative infinity + /// \retval FP_NAN for NaNs + /// \retval FP_NORMAL for all other (normal) values +// template typename enable::type fpclassify(T arg) { return functions::fpclassify(arg); } + inline int fpclassify(half arg) { return functions::fpclassify(arg); } + inline int fpclassify(expr arg) { return functions::fpclassify(arg); } + + /// Check if finite number. + /// \param arg number to check + /// \retval true if neither infinity nor NaN + /// \retval false else +// template typename enable::type isfinite(T arg) { return functions::isfinite(arg); } + inline bool isfinite(half arg) { return functions::isfinite(arg); } + inline bool isfinite(expr arg) { return functions::isfinite(arg); } + + /// Check for infinity. + /// \param arg number to check + /// \retval true for positive or negative infinity + /// \retval false else +// template typename enable::type isinf(T arg) { return functions::isinf(arg); } + inline bool isinf(half arg) { return functions::isinf(arg); } + inline bool isinf(expr arg) { return functions::isinf(arg); } + + /// Check for NaN. + /// \param arg number to check + /// \retval true for NaNs + /// \retval false else +// template typename enable::type isnan(T arg) { return functions::isnan(arg); } + inline bool isnan(half arg) { return functions::isnan(arg); } + inline bool isnan(expr arg) { return functions::isnan(arg); } + + /// Check if normal number. + /// \param arg number to check + /// \retval true if normal number + /// \retval false if either subnormal, zero, infinity or NaN +// template typename enable::type isnormal(T arg) { return functions::isnormal(arg); } + inline bool isnormal(half arg) { return functions::isnormal(arg); } + inline bool isnormal(expr arg) { return functions::isnormal(arg); } + + /// Check sign. + /// \param arg number to check + /// \retval true for negative number + /// \retval false for positive number +// template typename enable::type signbit(T arg) { return functions::signbit(arg); } + inline bool signbit(half arg) { return functions::signbit(arg); } + inline bool signbit(expr arg) { return functions::signbit(arg); } + + /// \} + /// \name Comparison + /// \{ + + /// Comparison for greater than. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater than \a y + /// \retval false else +// template typename enable::type isgreater(T x, U y) { return functions::isgreater(x, y); } + inline bool isgreater(half x, half y) { return functions::isgreater(x, y); } + inline bool isgreater(half x, expr y) { return functions::isgreater(x, y); } + inline bool isgreater(expr x, half y) { return functions::isgreater(x, y); } + inline bool isgreater(expr x, expr y) { return functions::isgreater(x, y); } + + /// Comparison for greater equal. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater equal \a y + /// \retval false else +// template typename enable::type isgreaterequal(T x, U y) { return functions::isgreaterequal(x, y); } + inline bool isgreaterequal(half x, half y) { return functions::isgreaterequal(x, y); } + inline bool isgreaterequal(half x, expr y) { return functions::isgreaterequal(x, y); } + inline bool isgreaterequal(expr x, half y) { return functions::isgreaterequal(x, y); } + inline bool isgreaterequal(expr x, expr y) { return functions::isgreaterequal(x, y); } + + /// Comparison for less than. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less than \a y + /// \retval false else +// template typename enable::type isless(T x, U y) { return functions::isless(x, y); } + inline bool isless(half x, half y) { return functions::isless(x, y); } + inline bool isless(half x, expr y) { return functions::isless(x, y); } + inline bool isless(expr x, half y) { return functions::isless(x, y); } + inline bool isless(expr x, expr y) { return functions::isless(x, y); } + + /// Comparison for less equal. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less equal \a y + /// \retval false else +// template typename enable::type islessequal(T x, U y) { return functions::islessequal(x, y); } + inline bool islessequal(half x, half y) { return functions::islessequal(x, y); } + inline bool islessequal(half x, expr y) { return functions::islessequal(x, y); } + inline bool islessequal(expr x, half y) { return functions::islessequal(x, y); } + inline bool islessequal(expr x, expr y) { return functions::islessequal(x, y); } + + /// Comarison for less or greater. + /// \param x first operand + /// \param y second operand + /// \retval true if either less or greater + /// \retval false else +// template typename enable::type islessgreater(T x, U y) { return functions::islessgreater(x, y); } + inline bool islessgreater(half x, half y) { return functions::islessgreater(x, y); } + inline bool islessgreater(half x, expr y) { return functions::islessgreater(x, y); } + inline bool islessgreater(expr x, half y) { return functions::islessgreater(x, y); } + inline bool islessgreater(expr x, expr y) { return functions::islessgreater(x, y); } + + /// Check if unordered. + /// \param x first operand + /// \param y second operand + /// \retval true if unordered (one or two NaN operands) + /// \retval false else +// template typename enable::type isunordered(T x, U y) { return functions::isunordered(x, y); } + inline bool isunordered(half x, half y) { return functions::isunordered(x, y); } + inline bool isunordered(half x, expr y) { return functions::isunordered(x, y); } + inline bool isunordered(expr x, half y) { return functions::isunordered(x, y); } + inline bool isunordered(expr x, expr y) { return functions::isunordered(x, y); } + + /// \name Casting + /// \{ + + /// Cast to or from half-precision floating point number. + /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted + /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do. + /// It uses the default rounding mode. + /// + /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types + /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler + /// error and casting between [half](\ref half_float::half)s is just a no-op. + /// \tparam T destination type (half or built-in arithmetic type) + /// \tparam U source type (half or built-in arithmetic type) + /// \param arg value to cast + /// \return \a arg converted to destination type + template T half_cast(U arg) { return half_caster::cast(arg); } + + /// Cast to or from half-precision floating point number. + /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted + /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do. + /// + /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types + /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler + /// error and casting between [half](\ref half_float::half)s is just a no-op. + /// \tparam T destination type (half or built-in arithmetic type) + /// \tparam R rounding mode to use. + /// \tparam U source type (half or built-in arithmetic type) + /// \param arg value to cast + /// \return \a arg converted to destination type + template T half_cast(U arg) { return half_caster::cast(arg); } + /// \} + } + + using detail::operator==; + using detail::operator!=; + using detail::operator<; + using detail::operator>; + using detail::operator<=; + using detail::operator>=; + using detail::operator+; + using detail::operator-; + using detail::operator*; + using detail::operator/; + using detail::operator<<; + using detail::operator>>; + + using detail::abs; + using detail::fabs; + using detail::fmod; + using detail::remainder; + using detail::remquo; + using detail::fma; + using detail::fmax; + using detail::fmin; + using detail::fdim; + using detail::nanh; + using detail::exp; + using detail::expm1; + using detail::exp2; + using detail::log; + using detail::log10; + using detail::log1p; + using detail::log2; + using detail::sqrt; + using detail::cbrt; + using detail::hypot; + using detail::pow; + using detail::sin; + using detail::cos; + using detail::tan; + using detail::asin; + using detail::acos; + using detail::atan; + using detail::atan2; + using detail::sinh; + using detail::cosh; + using detail::tanh; + using detail::asinh; + using detail::acosh; + using detail::atanh; + using detail::erf; + using detail::erfc; + using detail::lgamma; + using detail::tgamma; + using detail::ceil; + using detail::floor; + using detail::trunc; + using detail::round; + using detail::lround; + using detail::nearbyint; + using detail::rint; + using detail::lrint; +#if HALF_ENABLE_CPP11_LONG_LONG + using detail::llround; + using detail::llrint; +#endif + using detail::frexp; + using detail::ldexp; + using detail::modf; + using detail::scalbn; + using detail::scalbln; + using detail::ilogb; + using detail::logb; + using detail::nextafter; + using detail::nexttoward; + using detail::copysign; + using detail::fpclassify; + using detail::isfinite; + using detail::isinf; + using detail::isnan; + using detail::isnormal; + using detail::signbit; + using detail::isgreater; + using detail::isgreaterequal; + using detail::isless; + using detail::islessequal; + using detail::islessgreater; + using detail::isunordered; + + using detail::half_cast; +} + + +/// Extensions to the C++ standard library. +namespace std +{ + /// Numeric limits for half-precision floats. + /// Because of the underlying single-precision implementation of many operations, it inherits some properties from + /// `std::numeric_limits`. + template<> class numeric_limits : public numeric_limits + { + public: + /// Supports signed values. + static HALF_CONSTEXPR_CONST bool is_signed = true; + + /// Is not exact. + static HALF_CONSTEXPR_CONST bool is_exact = false; + + /// Doesn't provide modulo arithmetic. + static HALF_CONSTEXPR_CONST bool is_modulo = false; + + /// IEEE conformant. + static HALF_CONSTEXPR_CONST bool is_iec559 = true; + + /// Supports infinity. + static HALF_CONSTEXPR_CONST bool has_infinity = true; + + /// Supports quiet NaNs. + static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true; + + /// Supports subnormal values. + static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present; + + /// Rounding mode. + /// Due to the mix of internal single-precision computations (using the rounding mode of the underlying + /// single-precision implementation) with the rounding mode of the single-to-half conversions, the actual rounding + /// mode might be `std::round_indeterminate` if the default half-precision rounding mode doesn't match the + /// single-precision rounding mode. + static HALF_CONSTEXPR_CONST float_round_style round_style = (std::numeric_limits::round_style== + half_float::half::round_style) ? half_float::half::round_style : round_indeterminate; + + /// Significant digits. + static HALF_CONSTEXPR_CONST int digits = 11; + + /// Significant decimal digits. + static HALF_CONSTEXPR_CONST int digits10 = 3; + + /// Required decimal digits to represent all possible values. + static HALF_CONSTEXPR_CONST int max_digits10 = 5; + + /// Number base. + static HALF_CONSTEXPR_CONST int radix = 2; + + /// One more than smallest exponent. + static HALF_CONSTEXPR_CONST int min_exponent = -13; + + /// Smallest normalized representable power of 10. + static HALF_CONSTEXPR_CONST int min_exponent10 = -4; + + /// One more than largest exponent + static HALF_CONSTEXPR_CONST int max_exponent = 16; + + /// Largest finitely representable power of 10. + static HALF_CONSTEXPR_CONST int max_exponent10 = 4; + + /// Smallest positive normal value. + static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0400); } + + /// Smallest finite value. + static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0xFBFF); } + + /// Largest finite value. + static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7BFF); } + + /// Difference between one and next representable value. + static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x1400); } + + /// Maximum rounding error. + static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW + { return half_float::half(half_float::detail::binary, (round_style==std::round_to_nearest) ? 0x3800 : 0x3C00); } + + /// Positive infinity. + static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7C00); } + + /// Quiet NaN. + static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7FFF); } + + /// Signalling NaN. + static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7DFF); } + + /// Smallest positive subnormal value. + static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0001); } + }; + +#if HALF_ENABLE_CPP11_HASH + /// Hash function for half-precision floats. + /// This is only defined if C++11 `std::hash` is supported and enabled. + template<> struct hash //: unary_function + { + /// Type of function argument. + typedef half_float::half argument_type; + + /// Function return type. + typedef size_t result_type; + + /// Compute hash function. + /// \param arg half to hash + /// \return hash value + result_type operator()(argument_type arg) const + { return hash()(static_cast(arg.data_)&-(arg.data_!=0x8000)); } + }; +#endif +} + + +#undef HALF_CONSTEXPR +#undef HALF_CONSTEXPR_CONST +#undef HALF_NOEXCEPT +#undef HALF_NOTHROW +#ifdef HALF_POP_WARNINGS + #pragma warning(pop) + #undef HALF_POP_WARNINGS +#endif + +#endif diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 9e18772..fe36902 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -26,6 +26,7 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "half.hpp" #include "instructions.h" #include "ptx_ir.h" #include "opcodes.h" @@ -1711,7 +1712,7 @@ unsigned int saturatei(unsigned int a, unsigned int max) ptx_reg_t f2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) { - assert( from_width == 32); + half_float::half tmp_h; enum cuda_math::cudaRoundMode mode = cuda_math::cudaRoundZero; switch (rounding_mode) { @@ -1753,7 +1754,10 @@ ptx_reg_t f2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, } } else { switch ( to_width ) { - case 16: assert(0); break; + case 16: + tmp_h = half_float::half(x.f32); + y.f16 = tmp_h; + break; case 32: assert(0); break; // handled by f2f case 64: y.f64 = x.f32; @@ -1904,48 +1908,53 @@ ptx_reg_t u2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, ptx_reg_t f2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) { ptx_reg_t y; - switch ( rounding_mode ) { - case RZI_OPTION: - y.f32 = truncf(x.f32); - break; - case RNI_OPTION: -#if CUDART_VERSION >= 3000 - y.f32 = nearbyintf(x.f32); -#else - y.f32 = cuda_math::__internal_nearbyintf(x.f32); -#endif - break; - case RMI_OPTION: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = floorf(x.f32); - } - break; - case RPI_OPTION: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = ceilf(x.f32); - } - break; - default: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = x.f32; - } - break; - } -#if CUDART_VERSION >= 3000 - if (isnanf(y.f32)) -#else - if (cuda_math::__cuda___isnanf(y.f32)) -#endif - { - y.u32 = 0x7fffffff; - } else if (saturation_mode) { - y.f32 = cuda_math::__saturatef(y.f32); + if (from_width == 16){ + half_float::detail::uint16 val = x.u16; + y.f32 = half_float::detail::half2float(val); + }else{ + switch ( rounding_mode ) { + case RZI_OPTION: + y.f32 = truncf(x.f32); + break; + case RNI_OPTION: + #if CUDART_VERSION >= 3000 + y.f32 = nearbyintf(x.f32); + #else + y.f32 = cuda_math::__internal_nearbyintf(x.f32); + #endif + break; + case RMI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = floorf(x.f32); + } + break; + case RPI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = ceilf(x.f32); + } + break; + default: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = x.f32; + } + break; + } + #if CUDART_VERSION >= 3000 + if (isnanf(y.f32)) + #else + if (cuda_math::__cuda___isnanf(y.f32)) + #endif + { + y.u32 = 0x7fffffff; + } else if (saturation_mode) { + y.f32 = cuda_math::__saturatef(y.f32); + } } return y; @@ -1993,7 +2002,7 @@ ptx_reg_t (*g_cvt_fn[11][11])( ptx_reg_t x, unsigned from_width, unsigned to_wid { chop, NULL, zext, zext, chop, NULL, zext, zext, u2f, u2f, u2f}, { chop, chop, NULL, zext, chop, chop, NULL, zext, u2f, u2f, u2f}, { chop, chop, chop, NULL, chop, chop, chop, NULL, u2f, u2f, u2f}, - { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , NULL,f2x, f2x}, + { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , NULL,f2f, f2x}, { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x, f2f, f2x}, { d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x, d2x, d2d} }; @@ -2145,9 +2154,6 @@ void cvt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) unsigned rounding_mode = pI->rounding_mode(); unsigned saturation_mode = pI->saturation_mode(); - if ( to_type == F16_TYPE || from_type == F16_TYPE ) - abort(); - int to_sign, from_sign; size_t from_width, to_width; unsigned src_fmt = type_info_key::type_decode(from_type, from_width, from_sign); @@ -2195,7 +2201,6 @@ void cvt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) } - if ( g_cvt_fn[src_fmt][dst_fmt] != NULL ) { ptx_reg_t result = g_cvt_fn[src_fmt][dst_fmt](data,from_width,to_width,to_sign, rounding_mode, saturation_mode); data = result; -- cgit v1.3 From bc89f8c7ebc77db6a28f5600d17c120fe693e988 Mon Sep 17 00:00:00 2001 From: J Date: Wed, 8 Aug 2018 16:59:28 -0700 Subject: brev implementation --- src/cuda-sim/instructions.cc | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) (limited to 'src/cuda-sim/instructions.cc') diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index fe36902..109cfa7 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -1459,7 +1459,34 @@ void breakaddr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) assert(pI->has_pred() == false); // pdom analysis cannot handle if this instruction is predicated } -void brev_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void brev_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + unsigned msb; + switch(i_type){ + case B32_TYPE: + msb = 31; + for (unsigned i=0; i<=msb; i++) { + if((src1_data.u32 & (1 << i))) + data.u32 |= 1 << (msb - i); + } + break; + case B64_TYPE: + msb = 63; + for (unsigned i=0; i<=msb; i++) { + if((src1_data.u64 & (1 << i))) + data.u64 |= 1 << (msb - i); + } + break; + default: assert(0); + } + thread->set_operand_value(dst,data, i_type, thread, pI); +} void brkpt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } void call_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -- cgit v1.3 From cbe8e3ddbca3895a499882c31b080a58a7545d25 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Thu, 16 Aug 2018 14:10:55 -0700 Subject: fix alignment bug in parser, modified ptxjitconfig, minor fixes --- src/cuda-sim/cuda-sim.cc | 45 ++++++++++++++++++++++++++++++++++++++++++-- src/cuda-sim/instructions.cc | 8 ++++---- src/cuda-sim/ptx_ir.h | 1 + src/cuda-sim/ptx_parser.cc | 2 +- 4 files changed, 49 insertions(+), 7 deletions(-) (limited to 'src/cuda-sim/instructions.cc') diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 5b80616..b85ba95 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1232,7 +1232,6 @@ void function_info::ptx_jit_config(std::map mallocPt std::vector< std::pair > param_data; std::vector offsets; std::vector paramIsPointer; - char buff[1024]; char * gpgpusim_path = getenv("GPGPUSIM_ROOT"); assert(gpgpusim_path!=NULL); @@ -1241,6 +1240,47 @@ void function_info::ptx_jit_config(std::map mallocPt std::string command = std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; std::string filename(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/params.config" + std::to_string(counter)); + //initialize paramList + char buff[1024]; + std::string filename_c(filename+"_c"); + snprintf(buff,1024,"c++filt %s > %s", get_name().c_str(), filename_c.c_str()); + system(buff); + FILE *fp = fopen(filename_c.c_str(), "r"); + fgets(buff, 1024, fp); + fclose(fp); + std::string fn(buff); + size_t pos1, pos2; + pos1 = fn.find_last_of("("); + pos2 = fn.find(")", pos1); + assert(pos2>pos1&&pos1>0); + strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); + char *tok; + tok = strtok(buff, ","); + std::string tmp; + while(tok!=NULL){ + std::string param(tok); + if(param.find("<")!=std::string::npos){ + assert(param.find(">")==std::string::npos); + assert(param.find("*")==std::string::npos); + tmp = param; + } else { + if (tmp.length()>0){ + tmp = ""; + assert(param.find(">")!=std::string::npos); + assert(param.find("<")==std::string::npos); + assert(param.find("*")==std::string::npos); + } + printf("%s\n", param.c_str()); + if(param.find("*")!=std::string::npos){ + paramIsPointer.push_back(true); + }else{ + paramIsPointer.push_back(false); + } + } + tok = strtok(NULL, ","); + } + + for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { param_info &p = i->second; std::string name = p.get_name(); @@ -1249,8 +1289,9 @@ void function_info::ptx_jit_config(std::map mallocPt param_t param_value = p.get_value(); offsets.push_back((unsigned)p.get_offset()); - if(param_value.size==sizeof(void*) && mallocPtr_Size.find(*(unsigned long long*)param_value.pdata)!=mallocPtr_Size.end()){ + if (paramIsPointer[i->first] && (*(unsigned long long*)param_value.pdata != 0)){ //is pointer + assert(param_value.size==sizeof(void*)&&"MisID'd this param as pointer"); size_t array_size = 0; unsigned long long param_pointer = *(unsigned long long*)param_value.pdata; if(mallocPtr_Size.find(param_pointer)!=mallocPtr_Size.end()){ diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 109cfa7..31fc434 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -1471,17 +1471,17 @@ void brev_impl( const ptx_instruction *pI, ptx_thread_info *thread ) switch(i_type){ case B32_TYPE: msb = 31; - for (unsigned i=0; i<=msb; i++) { + for (unsigned i=0; i<=msb; i++) { if((src1_data.u32 & (1 << i))) data.u32 |= 1 << (msb - i); - } + } break; case B64_TYPE: msb = 63; - for (unsigned i=0; i<=msb; i++) { + for (unsigned i=0; i<=msb; i++) { if((src1_data.u64 & (1 << i))) data.u64 |= 1 << (msb - i); - } + } break; default: assert(0); } diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index e726ab9..768b3e7 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1297,6 +1297,7 @@ public: unsigned offset = 0; if (m_param_configs.size()>0){ unsigned offset_nom = m_param_configs.back().first + m_param_configs.back().second; + //ensure offset matches alignment requirements offset = offset_nom%alignment ? (offset_nom/alignment + 1) * alignment : offset_nom; } m_param_configs.push_back(std::pair(size, offset)); diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index 3f3485c..0ba9a5b 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -572,7 +572,7 @@ void add_function_arg() PTX_PARSE_DPRINTF("add_function_arg \"%s\"", g_last_symbol->name().c_str() ); g_func_info->add_arg(g_last_symbol); unsigned alignment = (g_alignment_spec==-1) ? g_size : g_alignment_spec; - assert(alignment<=8); + assert(alignment==1||alignment==2||alignment==4||alignment==8||alignment==16);//known valid alignment values g_func_info->add_config_param( g_size, alignment); } -- cgit v1.3 From 7a9c450e6b905af9ca6cdd3c7b79ad5aec535a5a Mon Sep 17 00:00:00 2001 From: Deval Shah Date: Fri, 9 Nov 2018 21:27:34 -0800 Subject: resolving merge conflicts --- libcuda/cuda_runtime_api.cc | 35 +--------------- src/abstract_hardware_model.h | 30 ++------------ src/cuda-sim/cuda-sim.cc | 94 ++++--------------------------------------- src/cuda-sim/instructions.cc | 56 -------------------------- src/gpgpu-sim/gpu-sim.cc | 25 ++---------- 5 files changed, 16 insertions(+), 224 deletions(-) (limited to 'src/cuda-sim/instructions.cc') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index 4b50e34..f00fe52 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -1425,40 +1425,7 @@ __host__ cudaError_t CUDARTAPI cudaLaunch( const char *hostFun ) dim3 gridDim = config.grid_dim(); dim3 blockDim = config.block_dim(); - gpgpu_t *gpu = context->get_device()->get_gpgpu(); - checkpoint *g_checkpoint; - g_checkpoint = new checkpoint(); - class memory_space *global_mem; - global_mem = gpu->get_global_memory(); - - if(gpu->resume_option ==1 && (grid->get_uid()==gpu->resume_kernel)) - { - - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", grid->get_uid()); - - g_checkpoint->load_global_mem(global_mem, f1name); - for (int i=0;iresume_CTA;i++) - grid->increment_cta_id(); - } - if(gpu->resume_option==1 && (grid->get_uid()resume_kernel)) - { - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", grid->get_uid()); - - g_checkpoint->load_global_mem(global_mem, f1name); - printf("Skipping kernel %d as resuming from kernel %d\n",grid->get_uid(),gpu->resume_kernel ); - g_cuda_launch_stack.pop_back(); - return g_last_cudaError = cudaSuccess; - - } - if(gpu->checkpoint_option==1 && (grid->get_uid()>gpu->checkpoint_kernel)) - { - printf("Skipping kernel %d as checkpoint from kernel %d\n",grid->get_uid(),gpu->checkpoint_kernel ); - g_cuda_launch_stack.pop_back(); - return g_last_cudaError = cudaSuccess; - - } + printf("GPGPU-Sim PTX: pushing kernel \'%s\' to stream %u, gridDim= (%u,%u,%u) blockDim = (%u,%u,%u) \n", kname.c_str(), stream?stream->get_uid():0, gridDim.x,gridDim.y,gridDim.z,blockDim.x,blockDim.y,blockDim.z ); stream_operation op(grid,g_ptx_sim_mode,stream); diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index 2350db4..45fba76 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -381,8 +381,7 @@ public: void get_pdom_stack_top_info( unsigned *pc, unsigned *rpc ) const; unsigned get_rp() const; void print(FILE *fp) const; - void resume(char * fname) ; - void print_checkpoint (FILE *fout) const; + protected: unsigned m_warp_id; @@ -502,28 +501,14 @@ public: const char* get_ptx_inst_debug_file() const { return g_ptx_inst_debug_file; } int get_ptx_inst_debug_thread_uid() const { return g_ptx_inst_debug_thread_uid; } unsigned get_texcache_linesize() const { return m_texcache_linesize; } - int get_checkpoint_option() const {return checkpoint_option; } - int get_checkpoint_kernel() const {return checkpoint_kernel; } - int get_checkpoint_CTA() const {return checkpoint_CTA; } - int get_resume_option() const {return resume_option; } - int get_resume_kernel() const {return resume_kernel; } - int get_resume_CTA() const {return resume_CTA; } - int get_checkpoint_CTA_t() const {return checkpoint_CTA_t; } - int get_checkpoint_insn_Y() const {return checkpoint_insn_Y; } + private: // PTX options int m_ptx_convert_to_ptxplus; int m_ptx_use_cuobjdump; int m_experimental_lib_support; unsigned m_ptx_force_max_capability; - int checkpoint_option; - int checkpoint_kernel; - int checkpoint_CTA; - int resume_option; - int resume_kernel; - int resume_CTA; - int checkpoint_CTA_t; - int checkpoint_insn_Y; + int g_ptx_inst_debug_to_file; char* g_ptx_inst_debug_file; int g_ptx_inst_debug_thread_uid; @@ -535,14 +520,7 @@ private: class gpgpu_t { public: gpgpu_t( const gpgpu_functional_sim_config &config ); - int checkpoint_option; - int checkpoint_kernel; - int checkpoint_CTA; - int resume_option; - int resume_kernel; - int resume_CTA; - int checkpoint_CTA_t; - int checkpoint_insn_Y; + void* gpu_malloc( size_t size ); void* gpu_mallocarray( size_t count ); void gpu_memset( size_t dst_start_addr, int c, size_t count ); diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 642e301..6a6b307 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -2125,8 +2125,7 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) //before we execute, we should do PDOM analysis for functional simulation scenario. function_info *kernel_func_info = kernel.entry(); const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel_func_info); - checkpoint *g_checkpoint; - g_checkpoint = new checkpoint(); + if (kernel_func_info->is_pdom_set()) { printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", kernel.name().c_str() ); @@ -2143,21 +2142,12 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) - int inst_count=50; - int cp_op= g_the_gpu->checkpoint_option; - int cp_CTA = g_the_gpu->checkpoint_CTA; - int cp_kernel= g_the_gpu->checkpoint_kernel; - cp_count= g_the_gpu->checkpoint_insn_Y; - cp_cta_resume= g_the_gpu->checkpoint_CTA_t; - int cta_launched =0; //we excute the kernel one CTA (Block) at the time, as synchronization functions work block wise while(!kernel.no_more_ctas_to_run()){ unsigned temp=kernel.get_next_cta_id_single(); - if(cp_op==0 || (cp_op==1 && cta_launched= 5000) launch_all_device_kernels(); #endif - } - else - { - kernel.increment_cta_id(); - } - cta_launched++; + } - if(cp_op==1) - { - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", kernel.get_uid() ); - g_checkpoint->store_global_mem(g_the_gpu->get_global_memory(), f1name , "%08x"); - } + @@ -2227,7 +2207,6 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) void functionalCoreSim::initializeCTA(unsigned ctaid_cp) { int ctaLiveThreads=0; - symbol_table * symtab= m_kernel->entry()->get_symtab(); for(int i=0; i< m_warp_count; i++){ m_warpAtBarrier[i]=false; @@ -2240,10 +2219,7 @@ void functionalCoreSim::initializeCTA(unsigned ctaid_cp) for(unsigned i=0; ithreads_per_cta();i++) { ptx_sim_init_thread(*m_kernel,&m_thread[i],0,i,m_kernel->threads_per_cta()-i,m_kernel->threads_per_cta(),this,0,i/m_warp_size,(gpgpu_t*)m_gpu, true); assert(m_thread[i]!=NULL && !m_thread[i]->is_done()); - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/thread_%d_0_reg.txt",i ); - if(cp_cta_resume==1) - m_thread[i]->resume_reg_thread(fname,symtab); + ctaLiveThreads++; } @@ -2266,40 +2242,25 @@ void functionalCoreSim::createWarp(unsigned warpId) char fname[2048]; snprintf(fname,2048,"checkpoint_files/warp_%d_0_simt.txt",warpId ); - if(cp_cta_resume==1) - { - unsigned pc,rpc; - m_simt_stack[warpId]->resume(fname); - m_simt_stack[warpId]->get_pdom_stack_top_info(&pc,&rpc); - for(int i=warpId*m_warp_size; iset_npc(pc); - m_thread[i]->update_pc(); - } - } m_liveThreadCount[warpId]= liveThreadsCount; } void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp) { - cp_count= m_gpu->checkpoint_insn_Y; - cp_cta_resume= m_gpu->checkpoint_CTA_t; + initializeCTA(ctaid_cp); - int count=0; + while(true){ bool someOneLive= false; bool allAtBarrier = true; for(unsigned i=0;i0 && count>inst_count && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cpcheckpoint_CTA_t) && m_gpu->checkpoint_option==1) - { - someOneLive=false; - break; - } + if(!someOneLive) break; if(allAtBarrier){ for(unsigned i=0;ientry()->get_symtab(); - - - unsigned ctaid =m_kernel->get_next_cta_id_single(); - if(m_gpu->checkpoint_option==1 && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cpcheckpoint_CTA_t)) - { - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/shared_mem_%d.txt",ctaid-1 ); - g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname , "%08x"); - for(int i=0; i<32*m_warp_count;i++) - { - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i,ctaid-1 ); - m_thread[i]->print_reg_thread(fname); - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i,ctaid-1 ); - g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name , "%08x"); - m_thread[i]->set_done(); - m_thread[i]->exitCore(); - m_thread[i]->registerExit(); - } - - for(int i=0;iprint_checkpoint(fp); - fclose(fp); - } - } } diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 31a33c6..f57a3f7 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -183,63 +183,7 @@ void ptx_thread_info::set_reg( const symbol *reg, const ptx_reg_t &value ) m_last_set_operand_value = value; } -void ptx_thread_info::print_reg_thread(char * fname) -{ - - FILE *fp= fopen(fname,"w"); - assert(fp!=NULL); - - int size = m_regs.size(); - - if(size>0) - { - reg_map_t reg = m_regs.back(); - - typename reg_map_t::const_iterator it; - for (it = reg.begin(); it != reg.end(); ++it) - { - const std::string &name = it->first->name(); - const std::string &dec= it->first->decl_location(); - unsigned size = it->first->get_size_in_bytes(); - fprintf(fp,"%s %llu %s %d\n",name.c_str(),it->second, dec.c_str(),size ); - - } - //m_regs.pop_back(); - } - fclose(fp); - - } - -void ptx_thread_info::resume_reg_thread(char * fname, symbol_table * symtab) -{ - - FILE * fp2 = fopen(fname, "r"); - assert(fp2!=NULL); - //m_regs.push_back( reg_map_t() ); - char line [ 200 ]; - while ( fgets ( line, sizeof line, fp2 ) != NULL ) - { - symbol *reg; - char * pch; - unsigned size; - pch = strtok (line," "); - char * name =pch; - reg= symtab->lookup(name); - ptx_reg_t data; - pch = strtok (NULL," "); - data = atoi(pch); - pch = strtok (NULL," "); - char * decl= pch; - pch = strtok (NULL," "); - size = atoi(pch); - - - m_regs.back()[reg] = data; - } - fclose ( fp2 ); -} - ptx_reg_t ptx_thread_info::get_reg( const symbol *reg ) { diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 79a6fcd..c706f23 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -1414,38 +1414,19 @@ void shader_core_ctx::issue_block2core( kernel_info_t &kernel ) // bind functional simulation state of threads to hardware resources (simulation) warp_set_t warps; unsigned nthreads_in_block= 0; - function_info *kernel_func_info = kernel.entry(); - symbol_table * symtab= kernel_func_info->get_symtab(); - unsigned ctaid= kernel.get_next_cta_id_single(); - checkpoint *g_checkpoint= new checkpoint(); + for (unsigned i = start_thread; iwarp_size; nthreads_in_block += ptx_sim_init_thread(kernel,&m_thread[i],m_sid,i,cta_size-(i-start_thread),m_config->n_thread_per_shader,this,free_cta_hw_id,warp_id,m_cluster->get_gpu()); m_threadState[i].m_active = true; - // load thread local memory and register file - if(m_gpu->resume_option==1 && kernel.get_uid()==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaidcheckpoint_CTA_t ) - { - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i%cta_size,ctaid ); - m_thread[i]->resume_reg_thread(fname,symtab); - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i%cta_size,ctaid); - g_checkpoint->load_global_mem(m_thread[i]->m_local_mem, f1name); - } - // + warps.set( warp_id ); } assert( nthreads_in_block > 0 && nthreads_in_block <= m_config->n_thread_per_shader); // should be at least one, but less than max m_cta_status[free_cta_hw_id]=nthreads_in_block; - if(m_gpu->resume_option==1 && kernel.get_uid()==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaidcheckpoint_CTA_t ) - { - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/shared_mem_%d.txt", ctaid); - - g_checkpoint->load_global_mem(m_thread[start_thread]->m_shared_mem, f1name); - } + // now that we know which warps are used in this CTA, we can allocate // resources for use in CTA-wide barrier operations m_barriers.allocate_barrier(free_cta_hw_id,warps); -- cgit v1.3 From 642818ae5ff61c1544bcce9e7ba2dd0aea47ea6a Mon Sep 17 00:00:00 2001 From: Deval Shah Date: Fri, 9 Nov 2018 21:29:18 -0800 Subject: Adding checkpoint support --- libcuda/cuda_runtime_api.cc | 35 +++++++++++++++- src/abstract_hardware_model.h | 30 ++++++++++++-- src/cuda-sim/cuda-sim.cc | 94 +++++++++++++++++++++++++++++++++++++++---- src/cuda-sim/instructions.cc | 56 ++++++++++++++++++++++++++ src/gpgpu-sim/gpu-sim.cc | 25 ++++++++++-- 5 files changed, 224 insertions(+), 16 deletions(-) (limited to 'src/cuda-sim/instructions.cc') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index f00fe52..4b50e34 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -1425,7 +1425,40 @@ __host__ cudaError_t CUDARTAPI cudaLaunch( const char *hostFun ) dim3 gridDim = config.grid_dim(); dim3 blockDim = config.block_dim(); - + gpgpu_t *gpu = context->get_device()->get_gpgpu(); + checkpoint *g_checkpoint; + g_checkpoint = new checkpoint(); + class memory_space *global_mem; + global_mem = gpu->get_global_memory(); + + if(gpu->resume_option ==1 && (grid->get_uid()==gpu->resume_kernel)) + { + + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", grid->get_uid()); + + g_checkpoint->load_global_mem(global_mem, f1name); + for (int i=0;iresume_CTA;i++) + grid->increment_cta_id(); + } + if(gpu->resume_option==1 && (grid->get_uid()resume_kernel)) + { + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", grid->get_uid()); + + g_checkpoint->load_global_mem(global_mem, f1name); + printf("Skipping kernel %d as resuming from kernel %d\n",grid->get_uid(),gpu->resume_kernel ); + g_cuda_launch_stack.pop_back(); + return g_last_cudaError = cudaSuccess; + + } + if(gpu->checkpoint_option==1 && (grid->get_uid()>gpu->checkpoint_kernel)) + { + printf("Skipping kernel %d as checkpoint from kernel %d\n",grid->get_uid(),gpu->checkpoint_kernel ); + g_cuda_launch_stack.pop_back(); + return g_last_cudaError = cudaSuccess; + + } printf("GPGPU-Sim PTX: pushing kernel \'%s\' to stream %u, gridDim= (%u,%u,%u) blockDim = (%u,%u,%u) \n", kname.c_str(), stream?stream->get_uid():0, gridDim.x,gridDim.y,gridDim.z,blockDim.x,blockDim.y,blockDim.z ); stream_operation op(grid,g_ptx_sim_mode,stream); diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index 45fba76..2350db4 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -381,7 +381,8 @@ public: void get_pdom_stack_top_info( unsigned *pc, unsigned *rpc ) const; unsigned get_rp() const; void print(FILE *fp) const; - + void resume(char * fname) ; + void print_checkpoint (FILE *fout) const; protected: unsigned m_warp_id; @@ -501,14 +502,28 @@ public: const char* get_ptx_inst_debug_file() const { return g_ptx_inst_debug_file; } int get_ptx_inst_debug_thread_uid() const { return g_ptx_inst_debug_thread_uid; } unsigned get_texcache_linesize() const { return m_texcache_linesize; } - + int get_checkpoint_option() const {return checkpoint_option; } + int get_checkpoint_kernel() const {return checkpoint_kernel; } + int get_checkpoint_CTA() const {return checkpoint_CTA; } + int get_resume_option() const {return resume_option; } + int get_resume_kernel() const {return resume_kernel; } + int get_resume_CTA() const {return resume_CTA; } + int get_checkpoint_CTA_t() const {return checkpoint_CTA_t; } + int get_checkpoint_insn_Y() const {return checkpoint_insn_Y; } private: // PTX options int m_ptx_convert_to_ptxplus; int m_ptx_use_cuobjdump; int m_experimental_lib_support; unsigned m_ptx_force_max_capability; - + int checkpoint_option; + int checkpoint_kernel; + int checkpoint_CTA; + int resume_option; + int resume_kernel; + int resume_CTA; + int checkpoint_CTA_t; + int checkpoint_insn_Y; int g_ptx_inst_debug_to_file; char* g_ptx_inst_debug_file; int g_ptx_inst_debug_thread_uid; @@ -520,7 +535,14 @@ private: class gpgpu_t { public: gpgpu_t( const gpgpu_functional_sim_config &config ); - + int checkpoint_option; + int checkpoint_kernel; + int checkpoint_CTA; + int resume_option; + int resume_kernel; + int resume_CTA; + int checkpoint_CTA_t; + int checkpoint_insn_Y; void* gpu_malloc( size_t size ); void* gpu_mallocarray( size_t count ); void gpu_memset( size_t dst_start_addr, int c, size_t count ); diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 6a6b307..642e301 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -2125,7 +2125,8 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) //before we execute, we should do PDOM analysis for functional simulation scenario. function_info *kernel_func_info = kernel.entry(); const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel_func_info); - + checkpoint *g_checkpoint; + g_checkpoint = new checkpoint(); if (kernel_func_info->is_pdom_set()) { printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", kernel.name().c_str() ); @@ -2142,12 +2143,21 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) + int inst_count=50; + int cp_op= g_the_gpu->checkpoint_option; + int cp_CTA = g_the_gpu->checkpoint_CTA; + int cp_kernel= g_the_gpu->checkpoint_kernel; + cp_count= g_the_gpu->checkpoint_insn_Y; + cp_cta_resume= g_the_gpu->checkpoint_CTA_t; + int cta_launched =0; //we excute the kernel one CTA (Block) at the time, as synchronization functions work block wise while(!kernel.no_more_ctas_to_run()){ unsigned temp=kernel.get_next_cta_id_single(); + if(cp_op==0 || (cp_op==1 && cta_launched= 5000) launch_all_device_kernels(); #endif - + } + else + { + kernel.increment_cta_id(); + } + cta_launched++; } - + if(cp_op==1) + { + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", kernel.get_uid() ); + g_checkpoint->store_global_mem(g_the_gpu->get_global_memory(), f1name , "%08x"); + } @@ -2207,6 +2227,7 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) void functionalCoreSim::initializeCTA(unsigned ctaid_cp) { int ctaLiveThreads=0; + symbol_table * symtab= m_kernel->entry()->get_symtab(); for(int i=0; i< m_warp_count; i++){ m_warpAtBarrier[i]=false; @@ -2219,7 +2240,10 @@ void functionalCoreSim::initializeCTA(unsigned ctaid_cp) for(unsigned i=0; ithreads_per_cta();i++) { ptx_sim_init_thread(*m_kernel,&m_thread[i],0,i,m_kernel->threads_per_cta()-i,m_kernel->threads_per_cta(),this,0,i/m_warp_size,(gpgpu_t*)m_gpu, true); assert(m_thread[i]!=NULL && !m_thread[i]->is_done()); - + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/thread_%d_0_reg.txt",i ); + if(cp_cta_resume==1) + m_thread[i]->resume_reg_thread(fname,symtab); ctaLiveThreads++; } @@ -2242,25 +2266,40 @@ void functionalCoreSim::createWarp(unsigned warpId) char fname[2048]; snprintf(fname,2048,"checkpoint_files/warp_%d_0_simt.txt",warpId ); + if(cp_cta_resume==1) + { + unsigned pc,rpc; + m_simt_stack[warpId]->resume(fname); + m_simt_stack[warpId]->get_pdom_stack_top_info(&pc,&rpc); + for(int i=warpId*m_warp_size; iset_npc(pc); + m_thread[i]->update_pc(); + } + } m_liveThreadCount[warpId]= liveThreadsCount; } void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp) { - + cp_count= m_gpu->checkpoint_insn_Y; + cp_cta_resume= m_gpu->checkpoint_CTA_t; initializeCTA(ctaid_cp); - + int count=0; while(true){ bool someOneLive= false; bool allAtBarrier = true; for(unsigned i=0;i0 && count>inst_count && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cpcheckpoint_CTA_t) && m_gpu->checkpoint_option==1) + { + someOneLive=false; + break; + } if(!someOneLive) break; if(allAtBarrier){ for(unsigned i=0;ientry()->get_symtab(); + + + unsigned ctaid =m_kernel->get_next_cta_id_single(); + if(m_gpu->checkpoint_option==1 && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cpcheckpoint_CTA_t)) + { + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/shared_mem_%d.txt",ctaid-1 ); + g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname , "%08x"); + for(int i=0; i<32*m_warp_count;i++) + { + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i,ctaid-1 ); + m_thread[i]->print_reg_thread(fname); + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i,ctaid-1 ); + g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name , "%08x"); + m_thread[i]->set_done(); + m_thread[i]->exitCore(); + m_thread[i]->registerExit(); + } + + for(int i=0;iprint_checkpoint(fp); + fclose(fp); + } + } } diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index f57a3f7..31a33c6 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -183,7 +183,63 @@ void ptx_thread_info::set_reg( const symbol *reg, const ptx_reg_t &value ) m_last_set_operand_value = value; } +void ptx_thread_info::print_reg_thread(char * fname) +{ + + FILE *fp= fopen(fname,"w"); + assert(fp!=NULL); + + int size = m_regs.size(); + + if(size>0) + { + reg_map_t reg = m_regs.back(); + + typename reg_map_t::const_iterator it; + for (it = reg.begin(); it != reg.end(); ++it) + { + const std::string &name = it->first->name(); + const std::string &dec= it->first->decl_location(); + unsigned size = it->first->get_size_in_bytes(); + fprintf(fp,"%s %llu %s %d\n",name.c_str(),it->second, dec.c_str(),size ); + + } + //m_regs.pop_back(); + } + fclose(fp); + + } + +void ptx_thread_info::resume_reg_thread(char * fname, symbol_table * symtab) +{ + + FILE * fp2 = fopen(fname, "r"); + assert(fp2!=NULL); + //m_regs.push_back( reg_map_t() ); + char line [ 200 ]; + while ( fgets ( line, sizeof line, fp2 ) != NULL ) + { + symbol *reg; + char * pch; + unsigned size; + pch = strtok (line," "); + char * name =pch; + reg= symtab->lookup(name); + ptx_reg_t data; + pch = strtok (NULL," "); + data = atoi(pch); + pch = strtok (NULL," "); + char * decl= pch; + pch = strtok (NULL," "); + size = atoi(pch); + + + m_regs.back()[reg] = data; + } + fclose ( fp2 ); +} + ptx_reg_t ptx_thread_info::get_reg( const symbol *reg ) { diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index c706f23..79a6fcd 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -1414,19 +1414,38 @@ void shader_core_ctx::issue_block2core( kernel_info_t &kernel ) // bind functional simulation state of threads to hardware resources (simulation) warp_set_t warps; unsigned nthreads_in_block= 0; - + function_info *kernel_func_info = kernel.entry(); + symbol_table * symtab= kernel_func_info->get_symtab(); + unsigned ctaid= kernel.get_next_cta_id_single(); + checkpoint *g_checkpoint= new checkpoint(); for (unsigned i = start_thread; iwarp_size; nthreads_in_block += ptx_sim_init_thread(kernel,&m_thread[i],m_sid,i,cta_size-(i-start_thread),m_config->n_thread_per_shader,this,free_cta_hw_id,warp_id,m_cluster->get_gpu()); m_threadState[i].m_active = true; - + // load thread local memory and register file + if(m_gpu->resume_option==1 && kernel.get_uid()==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaidcheckpoint_CTA_t ) + { + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i%cta_size,ctaid ); + m_thread[i]->resume_reg_thread(fname,symtab); + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i%cta_size,ctaid); + g_checkpoint->load_global_mem(m_thread[i]->m_local_mem, f1name); + } + // warps.set( warp_id ); } assert( nthreads_in_block > 0 && nthreads_in_block <= m_config->n_thread_per_shader); // should be at least one, but less than max m_cta_status[free_cta_hw_id]=nthreads_in_block; - + if(m_gpu->resume_option==1 && kernel.get_uid()==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaidcheckpoint_CTA_t ) + { + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/shared_mem_%d.txt", ctaid); + + g_checkpoint->load_global_mem(m_thread[start_thread]->m_shared_mem, f1name); + } // now that we know which warps are used in this CTA, we can allocate // resources for use in CTA-wide barrier operations m_barriers.allocate_barrier(free_cta_hw_id,warps); -- cgit v1.3