From 18e0b0614611edcb19ee0a3b315c7b45e50b5595 Mon Sep 17 00:00:00 2001 From: Ahmed ElTantawy Date: Wed, 4 Mar 2015 12:02:52 -0800 Subject: initial support for CUDA 5.0, 5.5, 6.0 to get template from SDK running --- src/cuda-sim/ptx_ir.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index b51406e..601a13d 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1229,12 +1229,12 @@ public: void param_to_shared( memory_space *shared_mem, symbol_table *symtab ); void list_param( FILE *fout ) const; - const struct gpgpu_ptx_sim_kernel_info* get_kernel_info () const + const struct gpgpu_ptx_sim_info* get_kernel_info () const { return &m_kernel_info; } - const void set_kernel_info (const struct gpgpu_ptx_sim_kernel_info &info) { + const void set_kernel_info (const struct gpgpu_ptx_sim_info &info) { m_kernel_info = info; m_kernel_info.ptx_version = 10*get_ptx_version().ver(); m_kernel_info.sm_target = get_ptx_version().target(); @@ -1282,7 +1282,7 @@ private: unsigned num_reconvergence_pairs; //Registers/shmem/etc. used (from ptxas -v), loaded from ___.ptxinfo along with ___.ptx - struct gpgpu_ptx_sim_kernel_info m_kernel_info; + struct gpgpu_ptx_sim_info m_kernel_info; symbol_table *m_symtab; -- cgit v1.3 From db5fe9600dd57ce59864b0f22c9d5407231ab5b1 Mon Sep 17 00:00:00 2001 From: Jin Wang Date: Fri, 3 Oct 2014 19:08:39 -0400 Subject: ADD: initial support for instruction group used by CDP --- cuobjdump_to_ptxplus/ptx_parser.h | 5 +++++ src/cuda-sim/ptx.y | 4 ++-- src/cuda-sim/ptx_ir.cc | 40 +++++++++++++++++++++++++++++++++++++++ src/cuda-sim/ptx_ir.h | 9 +++++++++ src/cuda-sim/ptx_parser.cc | 11 +++++++++++ src/cuda-sim/ptx_parser.h | 4 ++++ 6 files changed, 71 insertions(+), 2 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/cuobjdump_to_ptxplus/ptx_parser.h b/cuobjdump_to_ptxplus/ptx_parser.h index 1c96b46..418a733 100644 --- a/cuobjdump_to_ptxplus/ptx_parser.h +++ b/cuobjdump_to_ptxplus/ptx_parser.h @@ -110,6 +110,11 @@ void add_alignment_spec( int ) {PTX_PARSE_DPRINTF(" ");} void add_pragma( const char *a ) {PTX_PARSE_DPRINTF(" ");} void add_constptr(const char* identifier1, const char* identifier2, int offset) {PTX_PARSE_DPRINTF(" ");} +//Jin: handle instructino group for cdp +void start_inst_group(){PTX_PARSE_DPRINTF(" ");}; +void end_inst_group(){PTX_PARSE_DPRINTF(" ");}; + + /*non-dummy stuff below this point*/ extern cuobjdumpInstList *g_headerList; diff --git a/src/cuda-sim/ptx.y b/src/cuda-sim/ptx.y index 2f85213..c8208ea 100644 --- a/src/cuda-sim/ptx.y +++ b/src/cuda-sim/ptx.y @@ -270,8 +270,8 @@ statement_list: directive_statement { add_directive(); } | instruction_statement { add_instruction(); } | statement_list directive_statement { add_directive(); } | statement_list instruction_statement { add_instruction(); } - | statement_list statement_block - | statement_block + | statement_list {start_inst_group();} statement_block {end_inst_group();} + | {start_inst_group();} statement_block {end_inst_group();} ; directive_statement: variable_declaration SEMI_COLON diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index 751b3f4..915c623 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -90,6 +90,11 @@ symbol_table::symbol_table( const char *scope_name, unsigned entry_point, symbol m_const_next = 0; m_global_next = 0x100; m_local_next = 0; + m_tex_next = 0; + + //Jin: handle instruction group for cdp + m_inst_group_id = 0; + m_parent = parent; if ( m_parent ) { m_shared_next = m_parent->m_shared_next; @@ -170,6 +175,41 @@ void symbol_table::add_function( function_info *func, const char *filename, unsi m_symbols[ func->get_name() ] = s; } +//Jin: handle instruction group for cdp +symbol_table* symbol_table::start_inst_group() { + char inst_group_name[1024]; + snprintf(inst_group_name, 1024, "%s_inst_group_%u", m_scope_name.c_str(), m_inst_group_id); + + //previous added + assert(m_inst_group_symtab.find(std::string(inst_group_name)) == m_inst_group_symtab.end()); + symbol_table *sym_table = new symbol_table(inst_group_name, 3/*inst group*/, this ); + + sym_table->m_global_next = m_global_next; + sym_table->m_shared_next = m_shared_next; + sym_table->m_local_next = m_local_next; + sym_table->m_reg_allocator = m_reg_allocator; + sym_table->m_tex_next = m_tex_next; + sym_table->m_const_next = m_const_next; + + m_inst_group_symtab[std::string(inst_group_name)] = sym_table; + + return sym_table; +} + +symbol_table * symbol_table::end_inst_group() { + symbol_table * sym_table = m_parent; + + sym_table->m_global_next = m_global_next; + sym_table->m_shared_next = m_shared_next; + sym_table->m_local_next = m_local_next; + sym_table->m_reg_allocator = m_reg_allocator; + sym_table->m_tex_next = m_tex_next; + sym_table->m_const_next = m_const_next; + sym_table->m_inst_group_id++; + + return sym_table; +} + void register_ptx_function( const char *name, function_info *impl ); // either libcuda or libopencl bool symbol_table::add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **sym_table ) diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 601a13d..7325e5f 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -330,6 +330,11 @@ public: iterator const_iterator_end() { return m_consts.end();} void dump(); + + //Jin: handle instruction group for cdp + symbol_table* start_inst_group(); + symbol_table* end_inst_group(); + private: unsigned m_reg_allocator; unsigned m_shared_next; @@ -347,6 +352,10 @@ private: std::list m_consts; std::map m_function_info_lookup; std::map m_function_symtab_lookup; + + //Jin: handle instruction group for cdp + unsigned m_inst_group_id; + std::map m_inst_group_symtab; }; class operand_info { diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index 824714a..39257da 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -187,6 +187,17 @@ void add_function_name( const char *name ) g_global_symbol_table->add_function( g_func_info, g_filename, ptx_lineno ); } +//Jin: handle instruction group for cdp +void start_inst_group() { + PTX_PARSE_DPRINTF("start_instruction_group"); + g_current_symbol_table = g_current_symbol_table->start_inst_group(); +} + +void end_inst_group() { + PTX_PARSE_DPRINTF("end_instruction_group"); + g_current_symbol_table = g_current_symbol_table->end_inst_group(); +} + void add_directive() { PTX_PARSE_DPRINTF("add_directive"); diff --git a/src/cuda-sim/ptx_parser.h b/src/cuda-sim/ptx_parser.h index fef7635..32f3903 100644 --- a/src/cuda-sim/ptx_parser.h +++ b/src/cuda-sim/ptx_parser.h @@ -94,6 +94,10 @@ void change_operand_neg( ); void set_immediate_operand_type( ); void version_header(double a); +//Jin: handle instructino group for cdp +void start_inst_group(); +void end_inst_group(); + #define NON_ARRAY_IDENTIFIER 1 #define ARRAY_IDENTIFIER_NO_DIM 2 #define ARRAY_IDENTIFIER 3 -- cgit v1.3 From 70e02ee5283cb96f0edcb46a15edf0ab6e1d0697 Mon Sep 17 00:00:00 2001 From: Jin Wang Date: Mon, 13 Oct 2014 18:58:37 -0400 Subject: ADD: add cudaGetParameterBufferV2 and add cudaLaunchDeviceV2 implementation. Kernel launch to stream not yet implemented --- src/cuda-sim/Makefile | 3 +- src/cuda-sim/cuda-sim.cc | 10 +++ src/cuda-sim/cuda_device_runtime.cc | 175 ++++++++++++++++++++++++++++++++++++ src/cuda-sim/cuda_device_runtime.h | 7 ++ src/cuda-sim/instructions.cc | 12 ++- src/cuda-sim/ptx_ir.h | 4 +- src/cuda-sim/ptx_parser.cc | 14 +++ 7 files changed, 219 insertions(+), 6 deletions(-) create mode 100644 src/cuda-sim/cuda_device_runtime.cc create mode 100644 src/cuda-sim/cuda_device_runtime.h (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/src/cuda-sim/Makefile b/src/cuda-sim/Makefile index 166e256..f479294 100644 --- a/src/cuda-sim/Makefile +++ b/src/cuda-sim/Makefile @@ -62,7 +62,7 @@ ifeq ($(GNUC_CPP0X),1) endif endif -OBJS := $(OUTPUT_DIR)/ptx_parser.o $(OUTPUT_DIR)/ptx_loader.o $(OUTPUT_DIR)/cuda_device_printf.o $(OUTPUT_DIR)/instructions.o $(OUTPUT_DIR)/cuda-sim.o $(OUTPUT_DIR)/ptx_ir.o $(OUTPUT_DIR)/ptx_sim.o $(OUTPUT_DIR)/memory.o $(OUTPUT_DIR)/ptx-stats.o $(OUTPUT_DIR)/decuda_pred_table/decuda_pred_table.o $(OUTPUT_DIR)/ptx.tab.o $(OUTPUT_DIR)/lex.ptx_.o $(OUTPUT_DIR)/ptxinfo.tab.o $(OUTPUT_DIR)/lex.ptxinfo_.o +OBJS := $(OUTPUT_DIR)/ptx_parser.o $(OUTPUT_DIR)/ptx_loader.o $(OUTPUT_DIR)/cuda_device_printf.o $(OUTPUT_DIR)/instructions.o $(OUTPUT_DIR)/cuda-sim.o $(OUTPUT_DIR)/ptx_ir.o $(OUTPUT_DIR)/ptx_sim.o $(OUTPUT_DIR)/memory.o $(OUTPUT_DIR)/ptx-stats.o $(OUTPUT_DIR)/decuda_pred_table/decuda_pred_table.o $(OUTPUT_DIR)/ptx.tab.o $(OUTPUT_DIR)/lex.ptx_.o $(OUTPUT_DIR)/ptxinfo.tab.o $(OUTPUT_DIR)/lex.ptxinfo_.o $(OUTPUT_DIR)/cuda_device_runtime.o OPT += -DCUDART_VERSION=$(CUDART_VERSION) @@ -145,5 +145,6 @@ $(OUTPUT_DIR)/ptx_sim.o: $(OUTPUT_DIR)/ptx.tab.c $(OUTPUT_DIR)/cuda-sim.o: $(OUTPUT_DIR)/ptx.tab.c $(OUTPUT_DIR)/lex.ptxinfo_.o: $(OUTPUT_DIR)/ptx.tab.c $(OUTPUT_DIR)/lex.ptx_.o: $(OUTPUT_DIR)/ptx.tab.c +$(OUTPUT_DIR)/cuda_device_runtime.o: $(OUTPUT_DIR)/ptx.tab.c include $(OUTPUT_DIR)/Makefile.makedepend diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 715be98..4933029 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1065,6 +1065,16 @@ void function_info::add_param_data( unsigned argn, struct gpgpu_ptx_sim_arg *arg } } +unsigned function_info::get_args_aligned_size() { + unsigned int align_size = 4; // a word + unsigned int total_size = 0; + for(unsigned int i = 0; i < num_args(); i++) { + total_size += ((m_args[i]->get_size_in_bytes() + align_size - 1) / align_size) * align_size; + } + return total_size; +} + + void function_info::finalize( memory_space *param_mem ) { unsigned param_address = 0; diff --git a/src/cuda-sim/cuda_device_runtime.cc b/src/cuda-sim/cuda_device_runtime.cc new file mode 100644 index 0000000..937eec8 --- /dev/null +++ b/src/cuda-sim/cuda_device_runtime.cc @@ -0,0 +1,175 @@ +//Jin: cuda_device_runtime.cc +//Defines CUDA device runtime APIs for CDP support + +#include +#include + +#define __CUDA_RUNTIME_API_H__ + +#include +#include +#include "../gpgpu-sim/gpu-sim.h" +#include "cuda-sim.h" +#include "ptx_ir.h" +#include "../stream_manager.h" +#include "cuda_device_runtime.h" + +#define DEV_RUNTIME_REPORT(a) \ + if( g_debug_execution ) { \ + std::cout << __FILE__ << ", " << __LINE__ << ": " << a << "\n"; \ + std::cout.flush(); \ + } + +std::map g_cuda_device_launch_map; +struct CUstream_st * g_device_default_stream = NULL; +extern stream_manager *g_stream_manager; + +//Handling device runtime api: +//void * cudaGetParameterBufferV2(void *func, dim3 gridDimension, dim3 blockDimension, unsigned int sharedMemSize) +void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) +{ + DEV_RUNTIME_REPORT("Calling cudaGetParameterBufferV2"); + + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert( n_args == 4 ); + + function_info * child_kernel_entry; + struct dim3 gridDim, blockDim; + unsigned int sharedMem; + + for( unsigned arg=0; arg < n_args; arg ++ ) { + const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# + const symbol *formal_param = target_func->get_arg(arg); //cudaGetParameterBufferV2_param_# + unsigned size=formal_param->get_size_in_bytes(); + assert( formal_param->is_param_local() ); + assert( actual_param_op.is_param_local() ); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + + if(arg == 0) {//function_info* for the child kernel + unsigned long long buf; + assert(size == sizeof(function_info *)); + thread->m_local_mem->read(from_addr, size, &buf); + child_kernel_entry = (function_info *)buf; + assert(child_kernel_entry); + DEV_RUNTIME_REPORT("child kernel name " << child_kernel_entry->get_name()); + } + else if(arg == 1) { //dim3 gridDim for the child kernel + assert(size == sizeof(struct dim3)); + thread->m_local_mem->read(from_addr, size, & gridDim); + DEV_RUNTIME_REPORT("grid (" << gridDim.x << ", " << gridDim.y << ", " << gridDim.z << ")"); + } + else if(arg == 2) { //dim3 blockDim for the child kernel + assert(size == sizeof(struct dim3)); + thread->m_local_mem->read(from_addr, size, & blockDim); + DEV_RUNTIME_REPORT("block (" << blockDim.x << ", " << blockDim.y << ", " << blockDim.z << ")"); + } + else if(arg == 3) { //unsigned int sharedMem + assert(size == sizeof(unsigned int)); + thread->m_local_mem->read(from_addr, size, & sharedMem); + DEV_RUNTIME_REPORT("shared memory " << sharedMem); + } + } + + //get total child kernel argument size and malloc buffer in global memory + unsigned child_kernel_arg_size = child_kernel_entry->get_args_aligned_size(); + void * param_buffer = thread->get_gpu()->gpu_malloc(child_kernel_arg_size); + DEV_RUNTIME_REPORT("child kernel arg size total " << child_kernel_arg_size << ", parameter buffer allocated at " << param_buffer); + + //create child kernel_info_t and index it with parameter_buffer address + kernel_info_t * child_grid = new kernel_info_t(gridDim, blockDim, child_kernel_entry); + assert(g_cuda_device_launch_map.find(param_buffer) == g_cuda_device_launch_map.end()); + g_cuda_device_launch_map[param_buffer] = child_grid; + + //copy the buffer address to retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 + const symbol *formal_return = target_func->get_return_var(); //void * + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaGetParameterBufferV2 return value has size of " << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && return_size == sizeof(void *)); + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, ¶m_buffer, NULL, NULL); + +} + +//Handling device runtime api: +//cudaError_t cudaLaunchDeviceV2(void *parameterBuffer, cudaStream_t stream) +void gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) { + DEV_RUNTIME_REPORT("Calling cudaLaunchDeviceV2"); + + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert( n_args == 2 ); + + kernel_info_t * child_grid = NULL; + function_info * child_kernel_entry = NULL; + void * parameter_buffer; + struct CUstream_st * child_stream; + for( unsigned arg=0; arg < n_args; arg ++ ) { + const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# + const symbol *formal_param = target_func->get_arg(arg); //cudaLaunchDeviceV2_param_# + unsigned size=formal_param->get_size_in_bytes(); + assert( formal_param->is_param_local() ); + assert( actual_param_op.is_param_local() ); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + + if(arg == 0) {//paramter buffer for child kernel (in global memory) + //get parameter_buffer from the cudaDeviceLaunchV2_param0 + assert(size == sizeof(void *)); + thread->m_local_mem->read(from_addr, size, ¶meter_buffer); + assert((size_t)parameter_buffer >= GLOBAL_HEAP_START); + DEV_RUNTIME_REPORT("Parameter buffer locating at global memory " << parameter_buffer); + + //get child grid info through parameter_buffer address + assert(g_cuda_device_launch_map.find(parameter_buffer) != g_cuda_device_launch_map.end()); + child_grid = g_cuda_device_launch_map[parameter_buffer]; + child_kernel_entry = child_grid->entry(); + DEV_RUNTIME_REPORT("find child kernel " << child_kernel_entry->get_name()); + + //copy data in parameter_buffer to child kernel param memory + unsigned child_kernel_arg_size = child_kernel_entry->get_args_aligned_size(); + DEV_RUNTIME_REPORT("child_kernel_arg_size " << child_kernel_arg_size); + memory_space *child_kernel_param_mem = child_grid->get_param_memory(); + size_t param_start_address = 0; + for(unsigned n = 0; n < child_kernel_arg_size; n++) { + unsigned char one_byte; + thread->get_gpu()->get_global_memory()->read((size_t)parameter_buffer + n, 1, &one_byte); + child_kernel_param_mem->write(param_start_address + n, 1, &one_byte, NULL, NULL); + } + } + else if(arg == 1) { //cudaStream for the child kernel + assert(size == sizeof(cudaStream_t)); + thread->m_local_mem->read(from_addr, size, &child_stream); + if(child_stream == 0) { //default stream on device + if(!g_device_default_stream) { + //g_device_default_stream = new struct CUstream_st(); + //g_stream_manager->add_stream(g_device_default_stream); + } + child_stream = g_device_default_stream; + } +// DEV_RUNTIME_REPORT("launching child kernel to stream " << child_stream->get_uid()); + } + + } + + //launch child kernel + stream_operation op(child_grid, g_ptx_sim_mode, child_stream); +// g_stream_manager->push(op); +// g_cuda_device_launch_map.erase(parameter_buffer); + + //set retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 + const symbol *formal_return = target_func->get_return_var(); //cudaError_t + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaLaunchDeviceV2 return value has size of " << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size + && return_size == sizeof(cudaError_t)); + cudaError_t error = cudaSuccess; + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); + +} diff --git a/src/cuda-sim/cuda_device_runtime.h b/src/cuda-sim/cuda_device_runtime.h new file mode 100644 index 0000000..1b10407 --- /dev/null +++ b/src/cuda-sim/cuda_device_runtime.h @@ -0,0 +1,7 @@ +//Jin: cuda_device_runtime.h +//Defines CUDA device runtime APIs for CDP support + +#pragma once + +void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); +void gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 254427b..1c47ad3 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -41,6 +41,9 @@ #include "../gpgpu-sim/gpu-sim.h" #include "../gpgpu-sim/shader.h" +//Jin: include device runtime for CDP +#include "cuda_device_runtime.h" + #include unsigned ptx_instruction::g_num_ptx_inst_uid=0; @@ -149,8 +152,7 @@ ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_in } else if ( op.is_local() ) { result.u64 = op.get_symbol()->get_address(); } else if ( op.is_function_address() ) { - result.u64 = op.get_symbol()->get_pc()->get_start_PC(); - printf("Get pc for kernel function %u\n", op.get_symbol()->get_pc()->get_start_PC()); + result.u64 = (size_t)op.get_symbol()->get_pc(); } else { const char *name = op.name().c_str(); printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand type for %s\n", name ); @@ -1411,12 +1413,14 @@ void call_impl( const ptx_instruction *pI, ptx_thread_info *thread ) gpgpusim_cuda_vprintf(pI, thread, target_func); return; } + + //Jin: handle device runtime apis for CDP else if(fname == "cudaGetParameterBufferV2") { - printf("calling cudaGetParameterBufferV2\n"); + gpgpusim_cuda_getParameterBufferV2(pI, thread, target_func); return; } else if(fname == "cudaLaunchDeviceV2") { - printf("calling cudaLaunchDeviceV2\n"); + gpgpusim_cuda_launchDeviceV2(pI, thread, target_func); return; } diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 7325e5f..a7ca27e 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -699,7 +699,7 @@ public: } bool is_immediate_address() const { - return m_immediate_address; + return m_immediate_address; } bool is_literal() const { return m_type == int_t || @@ -1209,6 +1209,8 @@ public: { return m_args.size(); } + unsigned get_args_aligned_size(); + const symbol* get_arg( unsigned n ) const { assert( n < m_args.size() ); diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index 39257da..baa3bcd 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -120,6 +120,20 @@ symbol_table *init_parser( const char *ptx_filename ) #define DEF(X,Y) g_ptx_token_decode[X] = Y; #include "ptx_parser_decode.def" #undef DEF + g_ptx_token_decode[undefined_space] = "undefined_space"; + g_ptx_token_decode[undefined_space] = "undefined_space=0"; + g_ptx_token_decode[reg_space] = "reg_space"; + g_ptx_token_decode[local_space] = "local_space"; + g_ptx_token_decode[shared_space] = "shared_space"; + g_ptx_token_decode[param_space_unclassified] = "param_space_unclassified"; + g_ptx_token_decode[param_space_kernel] = "param_space_kernel"; + g_ptx_token_decode[param_space_local] = "param_space_local"; + g_ptx_token_decode[const_space] = "const_space"; + g_ptx_token_decode[tex_space] = "tex_space"; + g_ptx_token_decode[surf_space] = "surf_space"; + g_ptx_token_decode[global_space] = "global_space"; + g_ptx_token_decode[generic_space] = "generic_space"; + g_ptx_token_decode[instruction_space] = "instruction_space"; return g_global_symbol_table; } -- cgit v1.3 From dafeb411265dbc0228889fe97d85b00f71363f10 Mon Sep 17 00:00:00 2001 From: Jin Wang Date: Fri, 1 Jul 2016 06:54:46 -0400 Subject: MOD: compute child parameter size --- src/cuda-sim/cuda-sim.cc | 9 +++++++-- src/cuda-sim/ptx_ir.cc | 1 + src/cuda-sim/ptx_ir.h | 3 +++ 3 files changed, 11 insertions(+), 2 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 980afc8..f5d8a88 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1066,7 +1066,10 @@ void function_info::add_param_data( unsigned argn, struct gpgpu_ptx_sim_arg *arg } unsigned function_info::get_args_aligned_size() { - + + if(m_args_aligned_size >= 0) + return m_args_aligned_size; + unsigned param_address = 0; unsigned int total_size = 0; for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { @@ -1081,7 +1084,9 @@ unsigned function_info::get_args_aligned_size() { total_size += arg_size; } - return (total_size + 3) / 4 * 4; //final size aligned to word + m_args_aligned_size = (total_size + 3) / 4 * 4; //final size aligned to word + + return m_args_aligned_size; } diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index bdc8381..176eb14 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -1292,6 +1292,7 @@ function_info::function_info(int entry_point ) m_kernel_info.regs = 0; m_kernel_info.smem = 0; m_local_mem_framesize = 0; + m_args_aligned_size = -1; } unsigned function_info::print_insn( unsigned pc, FILE * fp ) const diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index a7ca27e..a54ae41 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -291,6 +291,9 @@ private: std::list m_initializer; static unsigned sm_next_uid; + + //parameter size for device kernels + int m_args_aligned_size; }; class symbol_table { -- cgit v1.3 From 06c5a3ddc979a78dd38bf0f74170e4919cd32fab Mon Sep 17 00:00:00 2001 From: Jin Wang Date: Sun, 3 Jul 2016 06:46:15 -0400 Subject: BUG: wrong declaration for m_args_aligned_size --- src/cuda-sim/ptx_ir.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index a54ae41..55b01fd 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -292,8 +292,6 @@ private: std::list m_initializer; static unsigned sm_next_uid; - //parameter size for device kernels - int m_args_aligned_size; }; class symbol_table { @@ -1302,6 +1300,9 @@ private: static std::vector s_g_pc_to_insn; // a direct mapping from PC to instruction static unsigned sm_next_uid; + + //parameter size for device kernels + int m_args_aligned_size; }; class arg_buffer_t { -- cgit v1.3 From 35cf76f383ec8de6de901bbbcd8fb478f69e46e4 Mon Sep 17 00:00:00 2001 From: sspenst Date: Wed, 6 Jul 2016 13:56:52 -0700 Subject: Added sstarr memory, which works the same as shared memory --- cuobjdump_to_ptxplus/ptx_parser.h | 1 + src/abstract_hardware_model.h | 1 + src/cuda-sim/cuda-sim.cc | 8 +++++ src/cuda-sim/instructions.cc | 65 +++++++++++++++++++++++++++++++++++++++ src/cuda-sim/ptx.l | 1 + src/cuda-sim/ptx.y | 2 ++ src/cuda-sim/ptx_ir.h | 6 ++++ src/cuda-sim/ptx_parser.cc | 14 +++++++++ src/cuda-sim/ptx_sim.cc | 1 + src/cuda-sim/ptx_sim.h | 1 + 10 files changed, 100 insertions(+) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/cuobjdump_to_ptxplus/ptx_parser.h b/cuobjdump_to_ptxplus/ptx_parser.h index 1c96b46..22377b2 100644 --- a/cuobjdump_to_ptxplus/ptx_parser.h +++ b/cuobjdump_to_ptxplus/ptx_parser.h @@ -58,6 +58,7 @@ enum _memory_space_t { reg_space, local_space, shared_space, + sstarr_space, param_space_unclassified, param_space_kernel, /* global to all threads in a kernel : read-only */ param_space_local, /* local to a thread : read-writable */ diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index b29f918..750eb6a 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -41,6 +41,7 @@ enum _memory_space_t { reg_space, local_space, shared_space, + sstarr_space, param_space_unclassified, param_space_kernel, /* global to all threads in a kernel : read-only */ param_space_local, /* local to a thread : read-writable */ diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 09e9a81..57da23f 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1407,6 +1407,7 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel, std::list &active_threads = kernel.active_threads(); static std::map shared_memory_lookup; + static std::map sstarr_memory_lookup; static std::map ptx_cta_lookup; static std::map > local_memory_lookup; @@ -1450,6 +1451,7 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel, //initializing new CTA ptx_cta_info *cta_info = NULL; memory_space *shared_mem = NULL; + memory_space *sstarr_mem = NULL; unsigned cta_size = kernel.threads_per_cta(); unsigned max_cta_per_sm = num_threads/cta_size; // e.g., 256 / 48 = 5 @@ -1466,6 +1468,9 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel, snprintf(buf,512,"shared_%u", sid); shared_mem = new memory_space_impl<16*1024>(buf,4); shared_memory_lookup[sm_idx] = shared_mem; + snprintf(buf,512,"sstarr_%u", sid); + sstarr_mem = new memory_space_impl<16*1024>(buf,4); + sstarr_memory_lookup[sm_idx] = sstarr_mem; cta_info = new ptx_cta_info(sm_idx); ptx_cta_lookup[sm_idx] = cta_info; } else { @@ -1474,6 +1479,7 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel, sm_idx, sid, max_cta_per_sm ); } shared_mem = shared_memory_lookup[sm_idx]; + sstarr_mem = sstarr_memory_lookup[sm_idx]; cta_info = ptx_cta_lookup[sm_idx]; cta_info->check_cta_thread_status_and_reset(); } @@ -1506,9 +1512,11 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel, thd->cpy_tid_to_reg(tid3d); thd->set_valid(); thd->m_shared_mem = shared_mem; + thd->m_sstarr_mem = sstarr_mem; function_info *finfo = thd->func_info(); symbol_table *st = finfo->get_symtab(); thd->func_info()->param_to_shared(thd->m_shared_mem,st); + thd->func_info()->param_to_shared(thd->m_sstarr_mem,st); thd->m_cta_info = cta_info; cta_info->add_thread(thd); thd->m_local_mem = local_mem; diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 36aa29f..4eb5ce3 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -130,6 +130,8 @@ ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_in result.u64 = sym->get_address() + op.get_addr_offset(); } else if ( op.is_shared() ) { result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); + } else if ( op.is_sstarr() ) { + result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); } else { const char *name = op.name().c_str(); printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory operand type for %s\n", name ); @@ -142,6 +144,8 @@ ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_in result.u64 = op.get_symbol()->get_address(); } else if ( op.is_shared() ) { result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_sstarr() ) { + result.u64 = op.get_symbol()->get_address(); } else if ( op.is_const() ) { result.u64 = op.get_symbol()->get_address(); } else if ( op.is_global() ) { @@ -2347,6 +2351,7 @@ void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand case surf_space: mem = thread->get_surf_memory(); break; case param_space_kernel: mem = thread->get_param_memory(); break; case shared_space: mem = thread->m_shared_mem; break; + case sstarr_space: mem = thread->m_sstarr_mem; break; case const_space: mem = thread->get_global_memory(); break; case generic_space: if( thread->get_ptx_version().ver() >= 2.0 ) { @@ -3736,7 +3741,67 @@ void sqrt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) void sst_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { + const operand_info &src1 = pI->src1(); + const operand_info &src3 = pI->src3(); //may be scalar or vector of regs + unsigned type = pI->get_type(); + ptx_reg_t addr_reg = thread->get_operand_value(src1, src1, type, thread, 1); + ptx_reg_t src3_data; + memory_space_t space = pI->get_space(); + + memory_space *mem = NULL; + addr_t addr = addr_reg.u32; + + decode_space(space,thread,src1,mem,addr); + + size_t size; + int t; + type_info_key::type_decode(type,size,t); + + src3_data = thread->get_operand_value(src3, src1, type, thread, 1); + mem->write(addr,size/8,&src3_data.s64,thread,pI); + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; + + printf("SST instruction found.\n"); + + /*const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + unsigned type = pI->get_type(); + ptx_reg_t addr_reg = thread->get_operand_value(src1, src1, type, thread, 1); + memory_space_t space = pI->get_space(); + + memory_space *mem = NULL; + addr_t addr = addr_reg.u32; + + decode_space(space,thread,src1,mem,addr); + + size_t size; + int t; + type_info_key::type_decode(type,size,t); + + ptx_reg_t src2_data = thread->get_operand_value(src2, src1, type, thread, 1); + ptx_reg_t src3_data = thread->get_operand_value(src3, src1, type, thread, 1); + mem->write(addr,size/8,&src3_data.s64,thread,pI);*/ + + /* + switch ( i_type ) { + case U32_TYPE: + data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF); + carry = (data.u64 & 0x100000000)>>32; + break; + case U64_TYPE: + data.u64 = src1_data.u64 + src2_data.u64; + break; + default: assert(0); break; + }*/ + + //thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry ); + //thread->m_last_effective_address = addr; + //thread->m_last_memory_space = space; } void ssy_impl( const ptx_instruction *pI, ptx_thread_info *thread ) diff --git a/src/cuda-sim/ptx.l b/src/cuda-sim/ptx.l index 49fd790..69349a0 100644 --- a/src/cuda-sim/ptx.l +++ b/src/cuda-sim/ptx.l @@ -177,6 +177,7 @@ breakaddr TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE; \.section TC; return SECTION_DIRECTIVE; \.shared TC; return SHARED_DIRECTIVE; \.sreg TC; return SREG_DIRECTIVE; +\.sstarr TC; return SSTARR_DIRECTIVE; \.struct TC; return STRUCT_DIRECTIVE; \.surf TC; return SURF_DIRECTIVE; /* not in PTX 2.1 */ \.target TC; return TARGET_DIRECTIVE; diff --git a/src/cuda-sim/ptx.y b/src/cuda-sim/ptx.y index 4de39d1..97f4ff2 100644 --- a/src/cuda-sim/ptx.y +++ b/src/cuda-sim/ptx.y @@ -64,6 +64,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %token SECTION_DIRECTIVE %token SHARED_DIRECTIVE %token SREG_DIRECTIVE +%token SSTARR_DIRECTIVE %token STRUCT_DIRECTIVE %token SURF_DIRECTIVE %token TARGET_DIRECTIVE @@ -339,6 +340,7 @@ addressable_spec: CONST_DIRECTIVE { add_space_spec(const_space,$1); } | LOCAL_DIRECTIVE { add_space_spec(local_space,0); } | PARAM_DIRECTIVE { add_space_spec(param_space_unclassified,0); } | SHARED_DIRECTIVE { add_space_spec(shared_space,0); } + | SSTARR_DIRECTIVE { add_space_spec(sstarr_space,0); } | SURF_DIRECTIVE { add_space_spec(surf_space,0); } | TEX_DIRECTIVE { add_space_spec(tex_space,0); } ; diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 601a13d..7724443 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -222,6 +222,7 @@ public: bool is_label() const { return m_is_label;} bool is_shared() const { return m_is_shared;} + bool is_sstarr() const { return m_is_sstarr;} bool is_const() const { return m_is_const;} bool is_global() const { return m_is_global;} bool is_local() const { return m_is_local;} @@ -279,6 +280,7 @@ private: bool m_address_valid; bool m_is_label; bool m_is_shared; + bool m_is_sstarr; bool m_is_const; bool m_is_global; bool m_is_local; @@ -313,10 +315,12 @@ public: void set_label_address( const symbol *label, unsigned addr ); unsigned next_reg_num() { return ++m_reg_allocator;} addr_t get_shared_next() { return m_shared_next;} + addr_t get_sstarr_next() { return m_sstarr_next;} addr_t get_global_next() { return m_global_next;} addr_t get_local_next() { return m_local_next;} addr_t get_tex_next() { return m_tex_next;} void alloc_shared( unsigned num_bytes ) { m_shared_next += num_bytes;} + void alloc_sstarr( unsigned num_bytes ) { m_sstarr_next += num_bytes;} void alloc_global( unsigned num_bytes ) { m_global_next += num_bytes;} void alloc_local( unsigned num_bytes ) { m_local_next += num_bytes;} void alloc_tex( unsigned num_bytes ) { m_tex_next += num_bytes;} @@ -333,6 +337,7 @@ public: private: unsigned m_reg_allocator; unsigned m_shared_next; + unsigned m_sstarr_next; unsigned m_const_next; unsigned m_global_next; unsigned m_local_next; @@ -703,6 +708,7 @@ public: } return m_value.m_symbolic->is_shared(); } + bool is_sstarr() const { return m_value.m_symbolic->is_sstarr();} bool is_const() const { return m_value.m_symbolic->is_const();} bool is_global() const { return m_value.m_symbolic->is_global();} bool is_local() const { return m_value.m_symbolic->is_local();} diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index 824714a..a53a8fe 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -415,6 +415,20 @@ void add_identifier( const char *identifier, int array_dim, unsigned array_ident g_last_symbol->set_address( addr+addr_pad ); g_current_symbol_table->alloc_shared( num_bits/8 + addr_pad ); break; + case sstarr_space: + printf("GPGPU-Sim PTX: allocating sstarr region for \"%s\" ", + identifier); + fflush(stdout); + assert( (num_bits%8) == 0 ); + addr = g_current_symbol_table->get_sstarr_next(); + addr_pad = pad_address(addr, num_bits/8, 128); + printf("from 0x%x to 0x%lx (sstarr memory space)\n", + addr+addr_pad, + addr+addr_pad + num_bits/8); + fflush(stdout); + g_last_symbol->set_address( addr+addr_pad ); + g_current_symbol_table->alloc_sstarr( num_bits/8 + addr_pad ); + break; case const_space: if( array_ident == ARRAY_IDENTIFIER_NO_DIM ) { printf("GPGPU-Sim PTX: deferring allocation of constant region for \"%s\" (need size information)\n", identifier ); diff --git a/src/cuda-sim/ptx_sim.cc b/src/cuda-sim/ptx_sim.cc index 09844ae..511e8d6 100644 --- a/src/cuda-sim/ptx_sim.cc +++ b/src/cuda-sim/ptx_sim.cc @@ -153,6 +153,7 @@ ptx_thread_info::ptx_thread_info( kernel_info_t &kernel ) m_last_memory_space = undefined_space; m_branch_taken = 0; m_shared_mem = NULL; + m_sstarr_mem = NULL; m_cta_info = NULL; m_local_mem = NULL; m_symbol_table = NULL; diff --git a/src/cuda-sim/ptx_sim.h b/src/cuda-sim/ptx_sim.h index f926e6d..c66b68c 100644 --- a/src/cuda-sim/ptx_sim.h +++ b/src/cuda-sim/ptx_sim.h @@ -424,6 +424,7 @@ public: memory_space_t m_last_memory_space; dram_callback_t m_last_dram_callback; memory_space *m_shared_mem; + memory_space *m_sstarr_mem; memory_space *m_local_mem; ptx_cta_info *m_cta_info; ptx_reg_t m_last_set_operand_value; -- cgit v1.3 From 68336f112117bcef5b943650819a6764e9ebf4ce Mon Sep 17 00:00:00 2001 From: sspenst Date: Wed, 24 Aug 2016 15:24:19 -0700 Subject: Added shfl instruction --- src/abstract_hardware_model.h | 1 + src/cuda-sim/cuda-sim.cc | 20 +++++++++++- src/cuda-sim/instructions.cc | 75 +++++++++++++++++++++++++++++++++++++++++++ src/cuda-sim/opcodes.def | 1 + src/cuda-sim/opcodes.h | 4 ++- src/cuda-sim/ptx.l | 6 ++++ src/cuda-sim/ptx.y | 8 +++++ src/cuda-sim/ptx_ir.cc | 6 ++++ src/cuda-sim/ptx_ir.h | 2 ++ src/cuda-sim/ptx_sim.cc | 21 ++++++++++++ src/cuda-sim/ptx_sim.h | 12 +++++++ 11 files changed, 154 insertions(+), 2 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index b29f918..d0c807d 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -1051,6 +1051,7 @@ class core_t { warp_inst_t getExecuteWarp(unsigned warpId); void get_pdom_stack_top_info( unsigned warpId, unsigned *pc, unsigned *rpc ) const; kernel_info_t * get_kernel_info(){ return m_kernel;} + class ptx_thread_info ** get_thread_info() { return m_thread; } unsigned get_warp_size() const { return m_warp_size; } void and_reduction(unsigned ctaid, unsigned barid, bool value) { reduction_storage[ctaid][barid] &= value; } void or_reduction(unsigned ctaid, unsigned barid, bool value) { reduction_storage[ctaid][barid] |= value; } diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 09e9a81..8bf4ec8 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -769,6 +769,10 @@ void ptx_instruction::set_opcode_and_latency() initiation_interval = dp_init[2]; op = SFU_OP; break; + case SHFL_OP: + latency = 32; + initiation_interval = 15; + break; default: break; } @@ -845,8 +849,10 @@ void ptx_instruction::pre_decode() switch ( get_opcode() ) { #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break; +#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break; #include "opcodes.def" #undef OP_DEF +#undef OP_W_DEF default: printf( "Execution error: Invalid opcode (0x%x)\n", get_opcode() ); break; @@ -1240,8 +1246,10 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id) } switch ( pI->get_opcode() ) { #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,this); op_classification = CLASSIFICATION; break; +#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,get_core(),inst); op_classification = CLASSIFICATION; break; #include "opcodes.def" #undef OP_DEF +#undef OP_W_DEF default: printf( "Execution error: Invalid opcode (0x%x)\n", pI->get_opcode() ); break; } delete pJ; @@ -1408,6 +1416,7 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel, static std::map shared_memory_lookup; static std::map ptx_cta_lookup; + static std::map ptx_warp_lookup; static std::map > local_memory_lookup; if ( *thread_info != NULL ) { @@ -1486,7 +1495,16 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel, kernel.increment_thread_id(); new_tid += tid; ptx_thread_info *thd = new ptx_thread_info(kernel); - + + ptx_warp_info *warp_info = NULL; + if ( ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end() ) { + warp_info = new ptx_warp_info(); // num_threads should be threads in the warp + ptx_warp_lookup[hw_warp_id] = warp_info; + } else { + warp_info = ptx_warp_lookup[hw_warp_id]; + } + thd->m_warp_info = warp_info; + memory_space *local_mem = NULL; std::map::iterator l = local_mem_lookup.find(new_tid); if ( l != local_mem_lookup.end() ) { diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 7b0f4fa..05ba78f 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -47,8 +47,10 @@ unsigned ptx_instruction::g_num_ptx_inst_uid=0; const char *g_opcode_string[NUM_OPCODES] = { #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR, +#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR, #include "opcodes.def" #undef OP_DEF +#undef OP_W_DEF }; void inst_not_implemented( const ptx_instruction * pI ) ; @@ -3516,6 +3518,79 @@ void set_impl( const ptx_instruction *pI, ptx_thread_info *thread ) } +void shfl_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) +{ + unsigned i_type = pI->get_type(); + int tid = inst.warp_id() * core->get_warp_size(); + ptx_thread_info *thread = core->get_thread_info()[tid]; + ptx_warp_info *warp_info = thread->m_warp_info; + int lane = warp_info->get_done_threads(); + thread = core->get_thread_info()[tid + lane]; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + int bval = (thread->get_operand_value(src2, dst, i_type, thread, 1)).u32; + int cval = (thread->get_operand_value(src3, dst, i_type, thread, 1)).u32; + int mask = cval >> 8; + cval &= 0x1F; + + int maxLane = (lane & mask) | (cval & ~mask); + int minLane = lane & mask; + + int src_idx; + int p; + switch(pI->shfl_op()) { + case UP_OPTION: + src_idx = lane - bval; + p = (src_idx >= maxLane); + break; + case DOWN_OPTION: + src_idx = lane + bval; + p = (src_idx <= maxLane); + break; + case BFLY_OPTION: + src_idx = lane ^ bval; + p = (src_idx <= maxLane); + break; + case IDX_OPTION: + src_idx = minLane | (bval & ~mask); + p = (src_idx <= maxLane); + break; + default: + printf("GPGPU-Sim PTX: ERROR: Unrecognized shfl option\n"); + assert(0); + break; + } + // copy from own lane + if (!p) src_idx = lane; + // copy input from lane src_idx + ptx_reg_t data; + /*if (inst.active(src_idx) && i_type == PRED_TYPE) { + ptx_thread_info *source = core->get_thread_info()[tid + src_idx]; + data = source->get_operand_value(src1, dst, i_type, source, 1); + data.pred = p; + } else { + printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for inactive/predicated-off threads in a warp\n"); + data.u32 = 0; + }*/ + if (inst.active(src_idx)) { + ptx_thread_info *source = core->get_thread_info()[tid + src_idx]; + data = source->get_operand_value(src1, dst, i_type, source, 1); + } + if (i_type == PRED_TYPE) { + data.pred = p; + } + thread->set_operand_value(dst, data, i_type, thread, pI); + + // keep track of the number of threads that have executed in the warp + warp_info->inc_done_threads(); + if (warp_info->get_done_threads() == inst.active_count()) { + warp_info->reset_done_threads(); + } +} + void shl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { ptx_reg_t a, b, d; diff --git a/src/cuda-sim/opcodes.def b/src/cuda-sim/opcodes.def index 2ee6976..e1b1422 100644 --- a/src/cuda-sim/opcodes.def +++ b/src/cuda-sim/opcodes.def @@ -98,6 +98,7 @@ OP_DEF(SAD_OP,sad_impl,"sad",1,1) OP_DEF(SELP_OP,selp_impl,"selp",1,1) OP_DEF(SETP_OP,setp_impl,"setp",1,1) OP_DEF(SET_OP,set_impl,"set",1,1) +OP_W_DEF(SHFL_OP,shfl_impl,"shfl",1,10) OP_DEF(SHL_OP,shl_impl,"shl",1,1) OP_DEF(SHR_OP,shr_impl,"shr",1,1) OP_DEF(SIN_OP,sin_impl,"sin",1,4) diff --git a/src/cuda-sim/opcodes.h b/src/cuda-sim/opcodes.h index 871091c..aa133da 100644 --- a/src/cuda-sim/opcodes.h +++ b/src/cuda-sim/opcodes.h @@ -30,9 +30,11 @@ enum opcode_t { #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP, +#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP, #include "opcodes.def" - NUM_OPCODES + NUM_OPCODES #undef OP_DEF +#undef OP_W_DEF }; enum special_regs { diff --git a/src/cuda-sim/ptx.l b/src/cuda-sim/ptx.l index 88ccf6a..8fac4ac 100644 --- a/src/cuda-sim/ptx.l +++ b/src/cuda-sim/ptx.l @@ -115,6 +115,7 @@ sad TC; ptx_lval.int_value = SAD_OP; return OPCODE; selp TC; ptx_lval.int_value = SELP_OP; return OPCODE; setp TC; ptx_lval.int_value = SETP_OP; return OPCODE; set TC; ptx_lval.int_value = SET_OP; return OPCODE; +shfl TC; ptx_lval.int_value = SHFL_OP; return OPCODE; shl TC; ptx_lval.int_value = SHL_OP; return OPCODE; shr TC; ptx_lval.int_value = SHR_OP; return OPCODE; sin TC; ptx_lval.int_value = SIN_OP; return OPCODE; @@ -329,6 +330,11 @@ breakaddr TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE; \.nc TC; return NC_OPTION; +\.up TC; return UP_OPTION; +\.down TC; return DOWN_OPTION; +\.bfly TC; return BFLY_OPTION; +\.idx TC; return IDX_OPTION; + \.popc TC; return ATOMIC_POPC; \.and TC; return ATOMIC_AND; \.or TC; return ATOMIC_OR; diff --git a/src/cuda-sim/ptx.y b/src/cuda-sim/ptx.y index 4de39d1..166b15d 100644 --- a/src/cuda-sim/ptx.y +++ b/src/cuda-sim/ptx.y @@ -194,6 +194,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %token WB_OPTION; %token WT_OPTION; %token NC_OPTION; +%token UP_OPTION; +%token DOWN_OPTION; +%token BFLY_OPTION; +%token IDX_OPTION; %type function_decl_header %type function_decl @@ -451,6 +455,10 @@ option: type_spec | WB_OPTION { add_option(WB_OPTION); } | WT_OPTION { add_option(WT_OPTION); } | NC_OPTION { add_option(NC_OPTION); } + | UP_OPTION { add_option(UP_OPTION); } + | DOWN_OPTION { add_option(DOWN_OPTION); } + | BFLY_OPTION { add_option(BFLY_OPTION); } + | IDX_OPTION { add_option(IDX_OPTION); } ; atomic_operation_spec: ATOMIC_AND { add_option(ATOMIC_AND); } diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index 2eccabc..4cfe1b9 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -1171,6 +1171,12 @@ ptx_instruction::ptx_instruction( int opcode, break; case NC_OPTION: break; + case UP_OPTION: + case DOWN_OPTION: + case BFLY_OPTION: + case IDX_OPTION: + m_shfl_op = last_ptx_inst_option; + break; default: assert(0); break; diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 601a13d..0abbc83 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -993,6 +993,7 @@ public: unsigned saturation_mode() const { return m_saturation_mode;} unsigned dimension() const { return m_geom_spec;} unsigned barrier_op() const {return m_barrier_op;} + unsigned shfl_op() const {return m_shfl_op;} enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot }; enum vote_mode_t vote_mode() const { return m_vote_mode; } @@ -1058,6 +1059,7 @@ private: unsigned m_compare_op; unsigned m_saturation_mode; unsigned m_barrier_op; + unsigned m_shfl_op; std::list m_scalar_type; memory_space_t m_space_spec; diff --git a/src/cuda-sim/ptx_sim.cc b/src/cuda-sim/ptx_sim.cc index 09844ae..a3e43aa 100644 --- a/src/cuda-sim/ptx_sim.cc +++ b/src/cuda-sim/ptx_sim.cc @@ -128,6 +128,26 @@ unsigned ptx_cta_info::get_sm_idx() const return m_sm_idx; } +ptx_warp_info::ptx_warp_info() +{ + reset_done_threads(); +} + +unsigned ptx_warp_info::get_done_threads() const +{ + return m_done_threads; +} + +void ptx_warp_info::inc_done_threads() +{ + m_done_threads++; +} + +void ptx_warp_info::reset_done_threads() +{ + m_done_threads = 0; +} + unsigned g_ptx_thread_info_uid_next=1; unsigned g_ptx_thread_info_delete_count=0; @@ -153,6 +173,7 @@ ptx_thread_info::ptx_thread_info( kernel_info_t &kernel ) m_last_memory_space = undefined_space; m_branch_taken = 0; m_shared_mem = NULL; + m_warp_info = NULL; m_cta_info = NULL; m_local_mem = NULL; m_symbol_table = NULL; diff --git a/src/cuda-sim/ptx_sim.h b/src/cuda-sim/ptx_sim.h index f926e6d..449511f 100644 --- a/src/cuda-sim/ptx_sim.h +++ b/src/cuda-sim/ptx_sim.h @@ -167,6 +167,17 @@ private: std::set m_dangling_pointers; }; +class ptx_warp_info { +public: + ptx_warp_info(); + unsigned get_done_threads() const; + void inc_done_threads(); + void reset_done_threads(); + +private: + unsigned m_done_threads; +}; + class symbol; struct stack_entry { @@ -425,6 +436,7 @@ public: dram_callback_t m_last_dram_callback; memory_space *m_shared_mem; memory_space *m_local_mem; + ptx_warp_info *m_warp_info; ptx_cta_info *m_cta_info; ptx_reg_t m_last_set_operand_value; -- cgit v1.3 From 742c4dc4c2c85329754043d38c60b2a37fefdaa1 Mon Sep 17 00:00:00 2001 From: Amruth Date: Fri, 23 Mar 2018 19:13:00 -0700 Subject: dynamic pdom analysis at runtime --- libcuda/cuda_runtime_api.cc | 19 +++++++++++++++---- src/cuda-sim/cuda-sim.cc | 12 ++++++++++++ src/cuda-sim/ptx_ir.cc | 26 ++++++++++++++++++++++++++ src/cuda-sim/ptx_ir.h | 5 ++++- 4 files changed, 57 insertions(+), 5 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index e7952e2..948d81d 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -1013,7 +1013,16 @@ __host__ cudaError_t CUDARTAPI cudaLaunch( const char *hostFun ) printf("\nGPGPU-Sim PTX: cudaLaunch for 0x%p (mode=%s) on stream %u\n", hostFun, g_ptx_sim_mode?"functional simulation":"performance simulation", stream?stream->get_uid():0 ); kernel_info_t *grid = gpgpu_cuda_ptx_sim_init_grid(hostFun,config.get_args(),config.grid_dim(),config.block_dim(),context); + //do dynamic PDOM analysis for performance simulation scenario std::string kname = grid->name(); + function_info *kernel_func_info = grid->entry(); + if (kernel_func_info->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", kname.c_str() ); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", kname.c_str() ); + kernel_func_info->do_pdom(); + kernel_func_info->set_pdom(); + } dim3 gridDim = config.grid_dim(); dim3 blockDim = config.block_dim(); printf("GPGPU-Sim PTX: pushing kernel \'%s\' to stream %u, gridDim= (%u,%u,%u) blockDim = (%u,%u,%u) \n", @@ -1400,7 +1409,7 @@ void extract_code_using_cuobjdump(){ const char *override_cuobjdump = getenv("CUOBJDUMP_SIM_FILE"); char fname[1024]; - if (override_cuobjdump == NULL) { + if ((override_cuobjdump == NULL) || (strlen(override_cuobjdump)==0)) { char command[1000]; std::string app_binary = get_app_binary(); @@ -1733,7 +1742,8 @@ cuobjdumpPTXSection* findPTXSection(const std::string identifier){ void cuobjdumpInit(){ CUctx_st *context = GPGPUSim_Context(); extract_code_using_cuobjdump(); //extract all the output of cuobjdump to _cuobjdump_*.* - if (getenv("CUOBJDUMP_SIM_FILE")==NULL){ + const char* pre_load = getenv("CUOBJDUMP_SIM_FILE"); + if (pre_load ==NULL || strlen(pre_load)==0){ cuobjdumpSectionList = pruneSectionList(cuobjdumpSectionList, context); cuobjdumpSectionList = mergeSections(cuobjdumpSectionList); } @@ -1772,12 +1782,13 @@ void cuobjdumpParseBinary(unsigned int handle){ if (max_capability > 20) printf("WARNING: No guarantee that PTX will be parsed for SM version %u\n", max_capability); cuobjdumpPTXSection* ptx = NULL; - if(getenv("CUOBJDUMP_SIM_FILE")==NULL) + const char* pre_load = getenv("CUOBJDUMP_SIM_FILE"); + if(pre_load==NULL || strlen(pre_load)==0) ptx = findPTXSection(fname); symbol_table *symtab; char *ptxcode; const char *override_ptx_name = getenv("PTX_SIM_KERNELFILE"); - if (override_ptx_name == NULL or getenv("PTX_SIM_USE_PTX_FILE") == NULL) { + if (override_ptx_name == NULL or getenv("PTX_SIM_USE_PTX_FILE") == NULL or strlen(getenv("PTX_SIM_USE_PTX_FILE"))==0) { ptxcode = readfile(ptx->getPTXfilename()); } else { printf("GPGPU-Sim PTX: overriding embedded ptx with '%s' (PTX_SIM_USE_PTX_FILE is set)\n", override_ptx_name); diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index f51f57d..39a04dd 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -257,6 +257,8 @@ void function_info::ptx_assemble() fflush(stdout); printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", m_name.c_str() ); + //disable pdom analysis here and do it at runtime +#if 0 create_basic_blocks(); connect_basic_blocks(); bool modified = false; @@ -280,6 +282,7 @@ void function_info::ptx_assemble() print_postdominators(); print_ipostdominators(); } +#endif printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", m_name.c_str() ); for ( unsigned ii=0; ii < n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions @@ -1801,6 +1804,15 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) //using a shader core object for book keeping, it is not needed but as most function built for performance simulation need it we use it here extern gpgpu_sim *g_the_gpu; + //before we execute, we should do PDOM analysis for functional simulation scenario. + function_info *kernel_func_info = kernel.entry(); + if (kernel_func_info->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", kernel.name().c_str() ); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", kernel.name().c_str() ); + kernel_func_info->do_pdom(); + kernel_func_info->set_pdom(); + } //we excute the kernel one CTA (Block) at the time, as synchronization functions work block wise while(!kernel.no_more_ctas_to_run()){ diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index 8ebdcf8..6a17eaf 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -575,6 +575,31 @@ bool function_info::connect_break_targets() //connecting break instructions with return modified; } +void function_info::do_pdom() { + create_basic_blocks(); + connect_basic_blocks(); + bool modified = false; + do { + find_dominators(); + find_idominators(); + modified = connect_break_targets(); + } while (modified == true); + + if ( g_debug_execution>=50 ) { + print_basic_blocks(); + print_basic_block_links(); + print_basic_block_dot(); + } + if ( g_debug_execution>=2 ) { + print_dominators(); + } + find_postdominators(); + find_ipostdominators(); + if ( g_debug_execution>=50 ) { + print_postdominators(); + print_ipostdominators(); + } +} void intersect( std::set &A, const std::set &B ) { // return intersection of A and B in A @@ -1305,6 +1330,7 @@ function_info::function_info(int entry_point ) m_kernel_info.smem = 0; m_local_mem_framesize = 0; m_args_aligned_size = -1; + pdom_done = false; //initialize it to false } unsigned function_info::print_insn( unsigned pc, FILE * fp ) const diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 9ad1571..26a2839 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1178,7 +1178,7 @@ public: //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 void find_ipostdominators( ); void print_ipostdominators(); - + void do_pdom(); //function to call pdom analysis unsigned get_num_reconvergence_pairs(); @@ -1274,6 +1274,8 @@ public: m_local_mem_framesize = sz; } bool is_entry_point() const { return m_entry_point; } + bool is_pdom_set() const { return pdom_done; } //return pdom flag + void set_pdom() { pdom_done = true; } //set pdom flag private: unsigned m_uid; @@ -1281,6 +1283,7 @@ private: bool m_entry_point; bool m_extern; bool m_assembled; + bool pdom_done; //flag to check whether pdom is completed or not std::string m_name; ptx_instruction **m_instr_mem; unsigned m_start_PC; -- cgit v1.3 From deee9038d3d67e60f106776be3dd0a846dd11df9 Mon Sep 17 00:00:00 2001 From: Tor Aamodt Date: Sun, 1 Apr 2018 17:31:14 -0700 Subject: fix regressions -- move call to pre_decode into do_pdom --- src/cuda-sim/cuda-sim.cc | 4 ++-- src/cuda-sim/ptx_ir.cc | 57 ++++++++++++++++++++++++++++-------------------- src/cuda-sim/ptx_ir.h | 2 ++ 3 files changed, 37 insertions(+), 26 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 987e3f2..dce35ca 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -252,7 +252,7 @@ void function_info::ptx_assemble() target.set_type(label_t); } } - + m_n = n; printf(" done.\n"); fflush(stdout); @@ -282,7 +282,6 @@ void function_info::ptx_assemble() print_postdominators(); print_ipostdominators(); } -#endif printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", m_name.c_str() ); for ( unsigned ii=0; ii < n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions @@ -293,6 +292,7 @@ void function_info::ptx_assemble() fflush(stdout); m_assembled = true; +#endif } addr_t shared_to_generic( unsigned smid, addr_t addr ) diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index 17e91df..be25dbe 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -577,30 +577,39 @@ bool function_info::connect_break_targets() //connecting break instructions with return modified; } -void function_info::do_pdom() { - create_basic_blocks(); - connect_basic_blocks(); - bool modified = false; - do { - find_dominators(); - find_idominators(); - modified = connect_break_targets(); - } while (modified == true); - - if ( g_debug_execution>=50 ) { - print_basic_blocks(); - print_basic_block_links(); - print_basic_block_dot(); - } - if ( g_debug_execution>=2 ) { - print_dominators(); - } - find_postdominators(); - find_ipostdominators(); - if ( g_debug_execution>=50 ) { - print_postdominators(); - print_ipostdominators(); - } +void function_info::do_pdom() +{ + create_basic_blocks(); + connect_basic_blocks(); + bool modified = false; + do { + find_dominators(); + find_idominators(); + modified = connect_break_targets(); + } while (modified == true); + + if ( g_debug_execution>=50 ) { + print_basic_blocks(); + print_basic_block_links(); + print_basic_block_dot(); + } + if ( g_debug_execution>=2 ) { + print_dominators(); + } + find_postdominators(); + find_ipostdominators(); + if ( g_debug_execution>=50 ) { + print_postdominators(); + print_ipostdominators(); + } + printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", m_name.c_str() ); + for ( unsigned ii=0; ii < m_n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions + ptx_instruction *pI = m_instr_mem[ii]; + pI->pre_decode(); + } + printf("GPGPU-Sim PTX: ... done pre-decoding instructions for \'%s\'.\n", m_name.c_str() ); + fflush(stdout); + m_assembled = true; } void intersect( std::set &A, const std::set &B ) { diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 26a2839..85b2a3b 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1308,6 +1308,8 @@ private: //parameter size for device kernels int m_args_aligned_size; + + addr_t m_n; // offset in m_instr_mem (used in do_pdom) }; class arg_buffer_t { -- cgit v1.3 From 8ea33c977b26cfe96beb98cdda289b81b8fda899 Mon Sep 17 00:00:00 2001 From: Amruth Date: Sat, 14 Apr 2018 17:17:11 -0700 Subject: solving alignment issue --- libcuda/cuda_runtime_api.cc | 1 + src/cuda-sim/cuda-sim.cc | 7 ++++++- src/cuda-sim/ptx.l | 5 +++-- src/cuda-sim/ptx_ir.h | 1 + 4 files changed, 11 insertions(+), 3 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index d6ce4e0..ef46f00 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -1923,6 +1923,7 @@ void cuobjdumpParseBinary(unsigned int handle){ if (capability > max_capability) max_capability = capability; } if (max_capability > 20) printf("WARNING: No guarantee that PTX will be parsed for SM version %u\n", max_capability); + if (max_capability == 0) max_capability=context->get_device()->get_gpgpu()->get_config().get_forced_max_capability(); cuobjdumpPTXSection* ptx = NULL; const char* pre_load = getenv("CUOBJDUMP_SIM_FILE"); diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index dce35ca..2c87031 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1139,8 +1139,13 @@ void function_info::finalize( memory_space *param_mem ) } // copy the parameter over word-by-word so that parameter that crosses a memory page can be copied over //Jin: copy parameter using aligned rules + const type_info *paramtype = param->type(); + int align_amount = paramtype->get_key().get_alignment_spec(); + align_amount = (align_amount == -1) ? size : align_amount; + param_address = (param_address + align_amount - 1) / align_amount * align_amount; //aligned + const size_t word_size = 4; - param_address = (param_address + size - 1) / size * size; //aligned with size + //param_address = (param_address + size - 1) / size * size; //aligned with size for (size_t idx = 0; idx < size; idx += word_size) { const char *pdata = reinterpret_cast(param_value.pdata) + idx; // cast to char * for ptr arithmetic param_mem->write(param_address + idx, word_size, pdata,NULL,NULL); diff --git a/src/cuda-sim/ptx.l b/src/cuda-sim/ptx.l index 1b5d7f6..908c5be 100644 --- a/src/cuda-sim/ptx.l +++ b/src/cuda-sim/ptx.l @@ -36,7 +36,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "ptx.tab.h" #include -char linebuf[1024]; +#define LINEBUF_SIZE (64*1024) +char linebuf[LINEBUF_SIZE]; unsigned col = 0; #define TC col+=strlen(ptx_text); #define CHECK_UNSIGNED \ @@ -384,7 +385,7 @@ breakaddr TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE; "//"[^\n]* TC; // eat single -\n.* col=0; strncpy(linebuf, yytext + 1, 1024); yyless( 1 ); +\n.* col=0; strncpy(linebuf, yytext + 1, LINEBUF_SIZE); yyless( 1 ); " " TC; "\t" TC; diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 85b2a3b..6731763 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -91,6 +91,7 @@ public: bool is_tex() const { return m_space_spec == tex_space;} bool is_func_addr() const { return m_is_function?true:false; } int scalar_type() const { return m_scalar_type_spec;} + int get_alignment_spec() const { return m_alignment_spec;} unsigned type_decode( size_t &size, int &t ) const; static unsigned type_decode( int type, size_t &size, int &t ); memory_space_t get_memory_space() const { return m_space_spec; } -- cgit v1.3 From 60017ca1ddbe844a93f631fe2b86bc4101850037 Mon Sep 17 00:00:00 2001 From: Amruth Date: Thu, 19 Apr 2018 18:13:44 -0700 Subject: Crash when array pointers are passed --- README | 27 ++++++++++++++++++++++++++- src/cuda-sim/instructions.cc | 4 +++- src/cuda-sim/ptx_ir.h | 13 +++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/README b/README index 543177c..bf5aa62 100644 --- a/README +++ b/README @@ -235,6 +235,14 @@ The documentation resides at doc/doxygen/html. Step 3: Run ============ +Before we run, we need to make sure the application's executable file is dynamically linked to CUDA runtime library. This can be done during compilation of your program by introducing the nvcc flag "--cudart shared" in makefile (quotes should be excluded). + +To confirm the same, type the follwoing command: + +ldd + +You should see that your application is using libcudart.so file in GPGPUSim directory. + Copy the contents of configs/QuadroFX5800/ or configs/GTX480/ to your application's working directory. These files configure the microarchitecture models to resemble the respective GPGPU architectures. @@ -348,7 +356,24 @@ identify any compile time or runtime errors that occur due to the code merging process. -** Debugging failing GPGPU-Sim Regressions ** +4. MISCELLANEOUS + +4.1 Speeding up the execution + +Some applications take several hours to execute on GPGPUSim. This is because the simulator has to dump the PTX, analyze them and get resource usage statistics. This can be avoided everytime we execute the program in the following way: + +Step 1: Execute the program by enabling “-save_embedded_ptx 1” in config file, execute the code and let cuobjdump command dump all necessary files. After this process, you will get 2 new files namely: _cuobjdump_complete_output_ and _1.ptx + +Step 2: Create new environment variables or include the below in your .bashrc file: + a. export PTX_SIM_USE_PTX_FILE=_1.ptx + b. export PTX_SIM_KERNELFILE=_1.ptx + c. export CUOBJDUMP_SIM_FILE=_cuobjdump_complete_output_ + +Step 3: Disable -save_embedded_ptx flag, execute the code again. This will skip the dumping by cuobjdump and directly goes to executing the program thus saving time. + + +4.2 Debugging failing GPGPU-Sim Regressions + Credits: Tor M Aamodt To debug failing GPGPU-Sim regression tests you need to run them locally. The fastest way to do this, assuming you are working with GPGPU-Sim versions more recent than the GPGPU-Sim dev branch circa March 28, 2018 (commit hash 2221d208a745a098a60b0d24c05007e92aaba092), is to install Docker. The instructions below were tested with Docker CE version 18.03 on Ubuntu and Mac OS. Docker will enable you to run the same set of regressions used by GPGPU-Sim when submitting a pull request to https://github.com/gpgpu-sim/gpgpu-sim_distribution and also allow you to log in and launch GPGPU-Sim in gdb so you can inspect failures. diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 0025c52..e53aaab 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -154,7 +154,9 @@ ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_in } else if ( op.is_local() ) { result.u64 = op.get_symbol()->get_address(); } else if ( op.is_function_address() ) { - result.u64 = (size_t)op.get_symbol()->get_pc(); + result.u64 = (size_t)op.get_symbol()->get_pc(); + } else if ( op.is_param_kernel()) { + result.u64 = op.get_symbol()->get_address(); } else { const char *name = op.name().c_str(); printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand type for %s\n", name ); diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 6731763..58d5f49 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -164,6 +164,7 @@ public: m_is_global = false; m_is_local = false; m_is_param_local = false; + m_is_param_kernel = false; m_is_tex = false; m_is_func_addr = false; m_reg_num_valid = false; @@ -177,6 +178,7 @@ public: if ( type ) m_is_global = type->get_key().is_global(); if ( type ) m_is_local = type->get_key().is_local(); if ( type ) m_is_param_local = type->get_key().is_param_local(); + if ( type ) m_is_param_kernel = type->get_key().is_param_kernel(); if ( type ) m_is_tex = type->get_key().is_tex(); if ( type ) m_is_func_addr = type->get_key().is_func_addr(); } @@ -227,6 +229,7 @@ public: bool is_global() const { return m_is_global;} bool is_local() const { return m_is_local;} bool is_param_local() const { return m_is_param_local; } + bool is_param_kernel() const { return m_is_param_kernel; } bool is_tex() const { return m_is_tex;} bool is_func_addr() const { return m_is_func_addr; } bool is_reg() const @@ -284,6 +287,7 @@ private: bool m_is_global; bool m_is_local; bool m_is_param_local; + bool m_is_param_kernel; bool m_is_tex; bool m_is_func_addr; unsigned m_reg_num; @@ -400,6 +404,8 @@ public: m_type = symbolic_t; } else if ( addr->is_param_local() ) { m_type = symbolic_t; + } else if ( addr->is_param_kernel() ) { + m_type = symbolic_t; } else if ( addr->is_tex() ) { m_type = symbolic_t; } else if ( addr->is_func_addr() ) { @@ -676,6 +682,13 @@ public: return m_value.m_symbolic->type()->get_key().is_param_local(); } + bool is_param_kernel() const + { + if ( m_type != symbolic_t ) + return false; + return m_value.m_symbolic->type()->get_key().is_param_kernel(); + } + bool is_vector() const { if ( m_vector) return true; -- cgit v1.3 From 373d64290239f3ed74d98b20494383f03fe189b6 Mon Sep 17 00:00:00 2001 From: Nathan Conrad Date: Sun, 22 Apr 2018 23:31:42 -0400 Subject: Some classes were referred to as a class and a struct (reported as clang warnings). This makes these consistent. --- libcuda/cuda_runtime_api.cc | 2 +- src/cuda-sim/cuda-sim.cc | 2 +- src/cuda-sim/ptx_ir.h | 2 +- src/debug.cc | 2 +- src/debug.h | 2 +- src/gpgpu-sim/dram.h | 4 ++-- src/gpgpu-sim/mem_fetch.cc | 2 +- src/gpgpu-sim/mem_fetch.h | 4 ++-- src/gpgpu-sim/shader.h | 4 ++-- src/stream_manager.h | 6 +++--- 10 files changed, 15 insertions(+), 15 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index 9bdb993..5ef6115 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -324,7 +324,7 @@ private: gpgpu_ptx_sim_arg_list_t m_args; }; -class _cuda_device_id *GPGPUSim_Init() +struct _cuda_device_id *GPGPUSim_Init() { static _cuda_device_id *the_device = NULL; if( !the_device ) { diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index a34b99b..9f24c69 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -2067,7 +2067,7 @@ struct rec_pts { int s_num_recon; }; -struct std::map g_rpts; +class std::map g_rpts; struct rec_pts find_reconvergence_points( function_info *finfo ) { diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 9ad1571..8750187 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -105,7 +105,7 @@ private: int m_is_function; bool m_is_non_arch_reg; - friend class type_info_key_compare; + friend struct type_info_key_compare; }; class symbol_table; diff --git a/src/debug.cc b/src/debug.cc index cfd7bb0..ae15760 100644 --- a/src/debug.cc +++ b/src/debug.cc @@ -222,7 +222,7 @@ void gpgpu_sim::gpgpu_debug() } } -bool thread_at_brkpt( ptx_thread_info *thread, const struct brk_pt &b ) +bool thread_at_brkpt( ptx_thread_info *thread, const class brk_pt &b ) { return b.is_equal(thread->get_location(),thread->get_uid()); } diff --git a/src/debug.h b/src/debug.h index 7c79f1e..1277494 100644 --- a/src/debug.h +++ b/src/debug.h @@ -87,7 +87,7 @@ extern int gpgpu_ptx_instruction_classification ; class ptx_thread_info; class ptx_instruction; -bool thread_at_brkpt( ptx_thread_info *thd_info, const struct brk_pt &b ); +bool thread_at_brkpt( ptx_thread_info *thd_info, const class brk_pt &b ); void hit_watchpoint( unsigned watchpoint_num, ptx_thread_info *thd, const ptx_instruction *pI ); #endif diff --git a/src/gpgpu-sim/dram.h b/src/gpgpu-sim/dram.h index a8bff14..15c63e7 100644 --- a/src/gpgpu-sim/dram.h +++ b/src/gpgpu-sim/dram.h @@ -87,7 +87,7 @@ struct bank_t unsigned int bkgrpindex; }; -struct mem_fetch; +class mem_fetch; class dram_t { @@ -178,7 +178,7 @@ private: unsigned int ave_mrqs_partial; unsigned int bwutil_partial; - struct memory_stats_t *m_stats; + class memory_stats_t *m_stats; class Stats* mrqq_Dist; //memory request queue inside DRAM friend class frfcfs_scheduler; diff --git a/src/gpgpu-sim/mem_fetch.cc b/src/gpgpu-sim/mem_fetch.cc index 580c051..729636d 100644 --- a/src/gpgpu-sim/mem_fetch.cc +++ b/src/gpgpu-sim/mem_fetch.cc @@ -39,7 +39,7 @@ mem_fetch::mem_fetch( const mem_access_t &access, unsigned wid, unsigned sid, unsigned tpc, - const class memory_config *config ) + const struct memory_config *config ) { m_request_uid = sm_next_mf_request_uid++; m_access = access; diff --git a/src/gpgpu-sim/mem_fetch.h b/src/gpgpu-sim/mem_fetch.h index c89edbb..de98748 100644 --- a/src/gpgpu-sim/mem_fetch.h +++ b/src/gpgpu-sim/mem_fetch.h @@ -55,7 +55,7 @@ public: unsigned wid, unsigned sid, unsigned tpc, - const class memory_config *config ); + const struct memory_config *config ); ~mem_fetch(); void set_status( enum mem_fetch_status status, unsigned long long cycle ); @@ -141,7 +141,7 @@ private: static unsigned sm_next_mf_request_uid; - const class memory_config *m_mem_config; + const struct memory_config *m_mem_config; unsigned icnt_flit_size; }; diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index bdd8dbe..ea8c019 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -286,7 +286,7 @@ typedef std::bitset warp_set_t; int register_bank(int regnum, int wid, unsigned num_banks, unsigned bank_warp_shift); class shader_core_ctx; -class shader_core_config; +struct shader_core_config; class shader_core_stats; enum scheduler_prioritization_type @@ -967,7 +967,7 @@ struct ifetch_buffer_t { unsigned m_warp_id; }; -class shader_core_config; +struct shader_core_config; class simd_function_unit { public: diff --git a/src/stream_manager.h b/src/stream_manager.h index 222a1b2..d3a804f 100644 --- a/src/stream_manager.h +++ b/src/stream_manager.h @@ -93,7 +93,7 @@ public: m_stream=stream; m_done=false; } - stream_operation( class CUevent_st *e, struct CUstream_st *stream ) + stream_operation( struct CUevent_st *e, struct CUstream_st *stream ) { m_kernel=NULL; m_type=stream_event; @@ -172,10 +172,10 @@ private: bool m_sim_mode; kernel_info_t *m_kernel; - class CUevent_st *m_event; + struct CUevent_st *m_event; }; -class CUevent_st { +struct CUevent_st { public: CUevent_st( bool blocking ) { -- cgit v1.3 From 68674d4ba230df0d3bf9f4e5b035f4cf9cfc185b Mon Sep 17 00:00:00 2001 From: negargoli93 Date: Sat, 12 May 2018 16:09:04 -0700 Subject: commit for eece527project --- src/abstract_hardware_model.h | 6 +- src/cuda-sim/cuda-sim.cc | 11 ++- src/cuda-sim/instructions.cc | 213 +++++++++++++++++++----------------------- src/cuda-sim/opcodes.def | 2 +- src/cuda-sim/ptx.l | 2 +- src/cuda-sim/ptx_ir.h | 25 +++++ src/gpgpu-sim/scoreboard.cc | 4 + src/gpgpusim_entrypoint.cc | 1 - 8 files changed, 136 insertions(+), 128 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index cdd9cf3..9dc58d4 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -750,7 +750,7 @@ public: }; // the maximum number of destination, source, or address uarch operands in a instruction -#define MAX_REG_OPERANDS 8 +#define MAX_REG_OPERANDS 32 struct dram_callback_t { dram_callback_t() { function=NULL; instruction=NULL; thread=NULL; } @@ -825,8 +825,8 @@ public: address_type reconvergence_pc; // -1 => not a branch, -2 => use function return address - unsigned out[4]; - unsigned in[4]; + unsigned out[8]; + unsigned in[8]; unsigned char is_vectorin; unsigned char is_vectorout; int pred; // predicate register number diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 54d8796..006738a 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -792,9 +792,9 @@ void ptx_instruction::set_opcode_and_latency() initiation_interval = dp_init[2]; op = SFU_OP; break; - case BSMAD_OP: - latency = int_precision/int_lane_width; - initiation_interval = int_init_precision/int_init_lane_width; + case MMA_OP: + latency = 64; + initiation_interval = 64; break; case SHFL_OP: latency = 32; @@ -1301,6 +1301,7 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id) *((warp_inst_t*)pJ) = inst; // copy active mask information pI = pJ; } + if((pI->get_opcode()!=MMA_OP)||((pI->get_opcode()==MMA_OP)&&(lane_id==0))){ switch ( pI->get_opcode() ) { #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,this); op_classification = CLASSIFICATION; break; #define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,get_core(),inst); op_classification = CLASSIFICATION; break; @@ -1308,7 +1309,7 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id) #undef OP_DEF #undef OP_W_DEF default: printf( "Execution error: Invalid opcode (0x%x)\n", pI->get_opcode() ); break; - } + }} delete pJ; pI = pI_saved; @@ -1930,7 +1931,7 @@ void functionalCoreSim::executeWarp(unsigned i, bool &allAtBarrier, bool & someO { if(!m_warpAtBarrier[i] && m_liveThreadCount[i]!=0){ warp_inst_t inst =getExecuteWarp(i); - execute_warp_inst_t(inst,i); + execute_warp_inst_t(inst,i); if(inst.isatomic()) inst.do_atomic(true); if(inst.op==BARRIER_OP || inst.op==MEMORY_BARRIER_OP ) m_warpAtBarrier[i]=true; updateSIMTStack( i, &inst ); diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 493e307..7903343 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -771,6 +771,17 @@ void add_impl( const ptx_instruction *pI, ptx_thread_info *thread ) unsigned i_type = pI->get_type(); src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + //unsigned warpId_aa,warp_size_aa; + //warpId_aa = pI->warp_id(); + //warp_size_aa=32; + //dim3 t=thread->get_tid(); + //unsigned tid_aa=warp_size_aa*warpId_aa; + + ptx_thread_info *thread2; + thread2=thread; + src1_data = thread2->get_operand_value(src1, dst, i_type, thread2, 1); + src2_data = thread2->get_operand_value(src2, dst, i_type, thread2, 1); + unsigned rounding_mode = pI->rounding_mode(); int orig_rm = fegetround(); @@ -1483,135 +1494,102 @@ unsigned trunc(unsigned num, unsigned precision) { return num; } -void bsmad_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) +void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) { - // operands: - // 0 = output - // 1 = input precision - // 2 = output precision - // 3 = buffer0 - // 4 = buffer1 - // 5 = buffer2 - // 6 = buffer3 - // 7 = synapse value - // 8 = output value + int i,j,k,thrd; + int row,offset; + printf("mmaWorld\n"); + ptx_reg_t matrix_a[16][16]; + ptx_reg_t matrix_b[16][16]; + ptx_reg_t matrix_c[16][16]; + ptx_reg_t matrix_d[16][16]; + ptx_reg_t src_data; + ptx_thread_info *thread; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); unsigned type = pI->get_type(); int tid = inst.warp_id_func() * core->get_warp_size(); - ptx_thread_info *thread = core->get_thread_info()[tid]; - const int ip = (thread->get_operand_value(src1, dst, type, thread, 1)).u32; - const int op = (thread->get_operand_value(src2, dst, type, thread, 1)).u32; - const int THREADS = inst.active_count(); - const int INBUFFERS = 4; - const int OUTBUFFERS = (((32/ip)*INBUFFERS) / (32/op)) + ((((32/ip)*INBUFFERS) % (32/op)) != 0); - if (OUTBUFFERS > THREADS) { - printf("GPGPU-Sim PTX: BSMAD ERROR - Number of output registers required (%d) is greater than the number available (%d)\n", OUTBUFFERS, THREADS); - abort(); - } - ptx_warp_info *warp_info = thread->m_warp_info; - warp_info->inc_done_threads(); + const operand_info &dst = pI->operand_lookup(0); + +//NOT WOR thread = core->get_thread_info()[tid]; +//NOT WOR const operand_info &src_a= pI->operand_lookup(1); +//NOT WOR src_data= (thread->get_operand_value(src_a, dst, type, thread, 1)); +//NOT WOR thread->set_operand_value(dst, src_data, type, thread, pI); + for (thrd=0; thrd < core->get_warp_size(); thrd++){ + row=thrd/2; + offset=8*(thrd%2); + thread = core->get_thread_info()[tid+thrd]; + printf("thread=%d:",thrd); + for(i=8;i<=31;i++){ + const operand_info &src_a= pI->operand_lookup(i); + src_data= (thread->get_operand_value(src_a, dst, type, thread, 1)); + printf("%f ",src_data.f32); + if(i<=15) + matrix_a[row][offset+(i)%8]=src_data; + else if((i>15)&&(i<=23)) + matrix_b[row][offset+(i)%8]=src_data; + else if(i>23) + matrix_c[row][offset+(i)%8]=src_data; + - // threads within the warp are executed sequentially by the simulator, store output in first four registers - if (warp_info->get_done_threads() <= OUTBUFFERS) { - unsigned buffer[THREADS][INBUFFERS]; - unsigned synapse[THREADS]; - unsigned output; - - // loop through all threads in the warp and get all data - for (unsigned i = 0, j = 0; i < core->get_warp_size(); i++) { - if (inst.active(i)) { - const operand_info &src3 = pI->operand_lookup(3); - const operand_info &src4 = pI->operand_lookup(4); - const operand_info &src5 = pI->operand_lookup(5); - const operand_info &src6 = pI->operand_lookup(6); - const operand_info &src7 = pI->operand_lookup(7); - const operand_info &src8 = pI->operand_lookup(8); - - thread = core->get_thread_info()[tid+i]; - // get buffer data and synapse data from each thread - buffer[j][0] = (thread->get_operand_value(src3, dst, type, thread, 1)).u32; - buffer[j][1] = (thread->get_operand_value(src4, dst, type, thread, 1)).u32; - buffer[j][2] = (thread->get_operand_value(src5, dst, type, thread, 1)).u32; - buffer[j][3] = (thread->get_operand_value(src6, dst, type, thread, 1)).u32; - synapse[j] = (thread->get_operand_value(src7, dst, type, thread, 1)).u32; - j++; - // get output data from the first 4 threads - if (j == warp_info->get_done_threads()) { - output = (thread->get_operand_value(src8, dst, type, thread, 1)).u32; - } - } } + printf("\n"); + } - // unpack registers, compute enough outputs to fill an output register - unsigned *unpacked_output = (unsigned*)calloc(32/op,sizeof(unsigned)); - unsigned buffer_data_start = (32/op)*(warp_info->get_done_threads()-1); - for (unsigned i = buffer_data_start; i < (32/op + buffer_data_start) && i < (32/ip)*INBUFFERS; i++) { - unsigned buf = i/(32/ip); - unsigned pos = i%(32/ip); - // sum values from the buffers - int sum = 0; - unsigned mask = (unsigned)(pow(2,ip)-1) << (pos*ip); - for (int j = 0; j < THREADS; j++) { - //sum += ((mask & buffer[j][buf]) >> (pos*ip)) * synapse[j]; - sum += trunc(((mask & buffer[j][buf]) >> (pos*ip)) * synapse[j], op); - } - // get the previous output - mask = (unsigned)(pow(2,op)-1) << (op*(i-buffer_data_start)); - int past_output = (mask & output) >> (op*(i-buffer_data_start)); - unpacked_output[i-buffer_data_start] = trunc(trunc(sum,op) + past_output,op); - // truncate sum, truncate (truncated sum + past_output) + printf("MATRIX_A\n"); + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + printf("%f ",matrix_a[i][j].f32); } - - // truncate output - /*for (unsigned i = 0; i < 32/op; i++) { - int mask = 1, latest_one = -1; - unsigned data = unpacked_output[i]; - for (unsigned j = 0; j < sizeof(unsigned)*8; j++) { - int bit = data & mask; - if (bit == 1) latest_one = j; - data >>= 1; - } - if (latest_one >= op) { - // round_up is 1 if the most significant truncated digit is a 1, otherwise it is 0 - int round_up = (unpacked_output[i] & (1 << (latest_one-op))) >> (latest_one-op); - unsigned shifted_output = unpacked_output[i] >> (latest_one-op+1); - // if shifted_output is a number like 1111, don't round up - if (shifted_output == (pow(2,op)-1)) round_up = 0; - unpacked_output[i] = shifted_output + round_up; - } - }*/ - - // pack the outputs into one register - unsigned mask = pow(2,op)-1; - unsigned output_data = 0; - for (int i = 0; i < 32/op; i++) { - output_data |= (unpacked_output[i] & mask) << (op*i); + printf("\n"); + } + printf("MATRIX_B\n"); + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + printf("%f ",matrix_b[i][j].f32); } - - // store the result in the correct thread's output register - for (unsigned i = 0, j = 0; i < core->get_warp_size(); i++) { - if (inst.active(i)) j++; - if (j == warp_info->get_done_threads()) { - thread = core->get_thread_info()[tid+i]; - ptx_reg_t data; - data.u32 = output_data; - thread->set_operand_value(dst, data, type, thread, pI); - break; + printf("\n"); + } + printf("MATRIX_C\n"); + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + printf("%f ",matrix_c[i][j].f32); + } + printf("\n"); + } + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + matrix_d[i][j].f32=0; + } + } + + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + for(k=0;k<16;k++){ + matrix_d[i][j].f32=matrix_d[i][j].f32+matrix_a[i][k].f32*matrix_b[j][k].f32; } + matrix_d[i][j].f32+=matrix_c[i][j].f32; } } - - // once the warp has finished, set the number of completed threads back to 0 for the next warp - if (warp_info->get_done_threads() == THREADS) { - warp_info->reset_done_threads(); + printf("MATRIX_D\n"); + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + printf("%f ",matrix_d[i][j].f32); + } + printf("\n"); + } + for (thrd=0; thrd < core->get_warp_size(); thrd++){ + thread = core->get_thread_info()[tid+thrd]; + row=thrd/2; + offset=8*(thrd%2); + for(i=0;i<8;i++){ + const operand_info &dst = pI->operand_lookup(i); + const symbol *r2; + r2=dst.get_symbol(); + printf("thrd=%d,i=%d,register%s, data=%f\n",thrd,i,(r2->name()).c_str(),matrix_d[row][offset+i].f32); + thread->set_operand_value(dst, matrix_d[row][offset+i], type, thread, pI); + } } - - // set the latency assuming 4 bits of each input get processed every cycle - // mutable latency variable??? - //pI->latency = (ip+3)/4; + } void call_impl( const ptx_instruction *pI, ptx_thread_info *thread ) @@ -4098,7 +4076,8 @@ void st_impl( const ptx_instruction *pI, ptx_thread_info *thread ) if (!vector_spec) { data = thread->get_operand_value(src1, dst, type, thread, 1); mem->write(addr,size/8,&data.s64,thread,pI); - } else { + printf("addr=%d data=%d\n",addr,data.s64); + } else { if (vector_spec == V2_TYPE) { ptx_reg_t* ptx_regs = new ptx_reg_t[2]; thread->get_vector_operand_values(src1, ptx_regs, 2); diff --git a/src/cuda-sim/opcodes.def b/src/cuda-sim/opcodes.def index 41f2f22..a3cc83f 100644 --- a/src/cuda-sim/opcodes.def +++ b/src/cuda-sim/opcodes.def @@ -52,7 +52,7 @@ OP_DEF(BRA_OP,bra_impl,"bra",0,3) OP_DEF(BRX_OP,brx_impl,"brx",0,3) OP_DEF(BREV_OP,brev_impl,"brev",1,1) OP_DEF(BRKPT_OP,brkpt_impl,"brkpt",1,9) -OP_W_DEF(BSMAD_OP,bsmad_impl,"bsmad",1,1) +OP_W_DEF(MMA_OP,mma_impl,"mma",1,1) OP_DEF(CALL_OP,call_impl,"call",1,3) OP_DEF(CALLP_OP,callp_impl,"callp",1,3) OP_DEF(CLZ_OP,clz_impl,"clz",1,1) diff --git a/src/cuda-sim/ptx.l b/src/cuda-sim/ptx.l index 7620134..e07e339 100644 --- a/src/cuda-sim/ptx.l +++ b/src/cuda-sim/ptx.l @@ -68,7 +68,7 @@ bra TC; ptx_lval.int_value = BRA_OP; return OPCODE; brx TC; ptx_lval.int_value = BRX_OP; return OPCODE; brev TC; ptx_lval.int_value = BREV_OP; return OPCODE; brkpt TC; ptx_lval.int_value = BRKPT_OP; return OPCODE; -bsmad TC; ptx_lval.int_value = BSMAD_OP; return OPCODE; +mma TC; ptx_lval.int_value = MMA_OP; return OPCODE; call TC; BEGIN(NOT_OPCODE); ptx_lval.int_value = CALL_OP; return OPCODE; // blocking opcode token in case the callee has the same name as an opcode callp TC; BEGIN(NOT_OPCODE); ptx_lval.int_value = CALLP_OP; return OPCODE; clz TC; ptx_lval.int_value = CLZ_OP; return OPCODE; diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 4c10373..0601b97 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -948,6 +948,31 @@ public: assert( m_operands.size() > 3 ); return m_operands[3]; } + const operand_info &src4() const + { + assert( m_operands.size() > 4 ); + return m_operands[4]; + } + const operand_info &src5() const + { + assert( m_operands.size() > 5 ); + return m_operands[5]; + } + const operand_info &src6() const + { + assert( m_operands.size() > 6 ); + return m_operands[6]; + } + const operand_info &src7() const + { + assert( m_operands.size() > 7 ); + return m_operands[7]; + } + const operand_info &src8() const + { + assert( m_operands.size() > 8 ); + return m_operands[8]; + } const operand_info &operand_lookup( unsigned n ) const { diff --git a/src/gpgpu-sim/scoreboard.cc b/src/gpgpu-sim/scoreboard.cc index f412054..b538fdf 100644 --- a/src/gpgpu-sim/scoreboard.cc +++ b/src/gpgpu-sim/scoreboard.cc @@ -146,6 +146,10 @@ bool Scoreboard::checkCollision( unsigned wid, const class inst_t *inst ) const if(inst->in[1] > 0) inst_regs.insert(inst->in[1]); if(inst->in[2] > 0) inst_regs.insert(inst->in[2]); if(inst->in[3] > 0) inst_regs.insert(inst->in[3]); + if(inst->in[3] > 0) inst_regs.insert(inst->in[4]); + if(inst->in[3] > 0) inst_regs.insert(inst->in[5]); + if(inst->in[3] > 0) inst_regs.insert(inst->in[6]); + if(inst->in[3] > 0) inst_regs.insert(inst->in[7]); if(inst->pred > 0) inst_regs.insert(inst->pred); if(inst->ar1 > 0) inst_regs.insert(inst->ar1); if(inst->ar2 > 0) inst_regs.insert(inst->ar2); diff --git a/src/gpgpusim_entrypoint.cc b/src/gpgpusim_entrypoint.cc index 04845e7..a6d7eb4 100644 --- a/src/gpgpusim_entrypoint.cc +++ b/src/gpgpusim_entrypoint.cc @@ -134,7 +134,6 @@ void *gpgpu_sim_thread_concurrent(void*) gpgpu_cuda_ptx_sim_main_func(*kernel); g_the_gpu->finish_functional_sim(kernel); } - //performance simulation if( g_the_gpu->active() ) { g_the_gpu->cycle(); -- cgit v1.3 From 7dfa2ae2e6f8ccaaf133318265a7ab00de546e82 Mon Sep 17 00:00:00 2001 From: aamir Date: Sun, 27 May 2018 14:18:53 -0700 Subject: added wmma parsing but execution getting aborted --- cuda-kernels/Makefile | 7 + cuda-kernels/_cuobjdump_1.elf | 15 + cuda-kernels/_cuobjdump_1.ptx | 170 ++++ cuda-kernels/_cuobjdump_1.sass | 2 + cuda-kernels/_cuobjdump_2.elf | 494 +++++++++ cuda-kernels/_cuobjdump_2.sass | 348 +++++++ cuda-kernels/_cuobjdump_complete_output_EIGzTK | 1055 ++++++++++++++++++++ cuda-kernels/_cuobjdump_complete_output_rndQyq | 1055 ++++++++++++++++++++ cuda-kernels/config_fermi_islip.icnt | 70 ++ cuda-kernels/gpgpu_inst_stats.txt | 1 + cuda-kernels/gpgpusim.config | 149 +++ ...usim_power_report__Sun-May-27-14-17-34-2018.log | 324 ++++++ ...usim_power_report__Sun-May-27-14-17-47-2018.log | 324 ++++++ cuda-kernels/gpuwattch_gtx1080Ti.xml | 538 ++++++++++ cuda-kernels/tensor_core | Bin 0 -> 48541 bytes cuda-kernels/tensor_core.cu | 250 +++++ cuobjdump_to_ptxplus/ptx_parser.h | 2 + src/cuda-sim/instructions.cc | 8 +- src/cuda-sim/opcodes.def | 2 + src/cuda-sim/opcodes.h | 11 +- src/cuda-sim/ptx.l | 18 +- src/cuda-sim/ptx.y | 10 + src/cuda-sim/ptx_ir.cc | 6 +- src/cuda-sim/ptx_ir.h | 33 + src/cuda-sim/ptx_parser.cc | 33 +- src/cuda-sim/ptx_parser.h | 2 + 26 files changed, 4918 insertions(+), 9 deletions(-) create mode 100755 cuda-kernels/Makefile create mode 100644 cuda-kernels/_cuobjdump_1.elf create mode 100644 cuda-kernels/_cuobjdump_1.ptx create mode 100644 cuda-kernels/_cuobjdump_1.sass create mode 100644 cuda-kernels/_cuobjdump_2.elf create mode 100644 cuda-kernels/_cuobjdump_2.sass create mode 100644 cuda-kernels/_cuobjdump_complete_output_EIGzTK create mode 100644 cuda-kernels/_cuobjdump_complete_output_rndQyq create mode 100755 cuda-kernels/config_fermi_islip.icnt create mode 100755 cuda-kernels/gpgpu_inst_stats.txt create mode 100755 cuda-kernels/gpgpusim.config create mode 100644 cuda-kernels/gpgpusim_power_report__Sun-May-27-14-17-34-2018.log create mode 100644 cuda-kernels/gpgpusim_power_report__Sun-May-27-14-17-47-2018.log create mode 100755 cuda-kernels/gpuwattch_gtx1080Ti.xml create mode 100755 cuda-kernels/tensor_core create mode 100644 cuda-kernels/tensor_core.cu (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/cuda-kernels/Makefile b/cuda-kernels/Makefile new file mode 100755 index 0000000..51a7760 --- /dev/null +++ b/cuda-kernels/Makefile @@ -0,0 +1,7 @@ +all: tensor_core.cu + nvcc -arch=sm_70 -lcudart -g -o tensor_core tensor_core.cu + +.PHONY: +clean: + rm tensorcore +# nvcc -arch=sm_70 --gpu-architecture=compute_50 --gpu-code=compute_50 -lcudart -g -o tensor_core tensor_core.cu diff --git a/cuda-kernels/_cuobjdump_1.elf b/cuda-kernels/_cuobjdump_1.elf new file mode 100644 index 0000000..672b0f0 --- /dev/null +++ b/cuda-kernels/_cuobjdump_1.elf @@ -0,0 +1,15 @@ +64bit elf: type=2, abi=7, sm=70, toolkit=90, flags = 0x460546 +Sections: +Index Offset Size ES Align Type Flags Link Info Name + 1 40 32 0 1 STRTAB 0 0 0 .shstrtab + 2 72 32 0 1 STRTAB 0 0 0 .strtab + 3 a8 18 18 8 SYMTAB 0 2 0 .symtab + +.section .strtab + +.section .shstrtab + +.section .symtab + index value size info other shndx name + 0 0 0 0 0 0 (null) + diff --git a/cuda-kernels/_cuobjdump_1.ptx b/cuda-kernels/_cuobjdump_1.ptx new file mode 100644 index 0000000..3453f4a --- /dev/null +++ b/cuda-kernels/_cuobjdump_1.ptx @@ -0,0 +1,170 @@ + + + + + + + +.version 6.0 +.target sm_70 +.address_size 64 + + +.extern .func (.param .b32 func_retval0) vprintf +( +.param .b64 vprintf_param_0, +.param .b64 vprintf_param_1 +) +; +.global .align 16 .b8 $str[9] = {99, 108, 111, 99, 107, 61, 37, 100, 0}; + +.visible .entry _Z12wmma_exampleP6__halfS0_Pfiiiff( +.param .u64 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_0, +.param .u64 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_1, +.param .u64 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_2, +.param .u32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_3, +.param .u32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_4, +.param .u32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_5, +.param .f32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_6, +.param .f32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_7 +) +{ +.local .align 8 .b8 __local_depot0[8]; +.reg .b64 %SP; +.reg .b64 %SPL; +.reg .pred %p<6>; +.reg .f32 %f<34>; +.reg .b32 %r<38>; +.reg .b64 %rd<18>; + + +mov.u64 %rd17, __local_depot0; +cvta.local.u64 %SP, %rd17; +ld.param.u64 %rd1, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_0]; +ld.param.u64 %rd2, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_1]; +ld.param.u64 %rd3, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_2]; +ld.param.u32 %r4, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_3]; +ld.param.u32 %r7, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_4]; +ld.param.u32 %r5, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_5]; + + mov.u32 %r6, %clock; + + mov.u32 %r8, %ntid.x; +mov.u32 %r9, %ctaid.x; +mov.u32 %r10, %tid.x; +mad.lo.s32 %r11, %r8, %r9, %r10; +mov.u32 %r12, WARP_SZ; +div.u32 %r13, %r11, %r12; +mov.u32 %r14, %ntid.y; +mov.u32 %r15, %ctaid.y; +mov.u32 %r16, %tid.y; +mad.lo.s32 %r17, %r14, %r15, %r16; +shl.b32 %r2, %r13, 4; +shl.b32 %r3, %r17, 4; +setp.lt.s32 %p1, %r2, %r4; +setp.gt.s32 %p2, %r5, 0; +and.pred %p3, %p1, %p2; +setp.lt.s32 %p4, %r3, %r7; +and.pred %p5, %p3, %p4; +mov.f32 %f26, 0f00000000; +mov.f32 %f27, %f26; +mov.f32 %f28, %f26; +mov.f32 %f29, %f26; +mov.f32 %f30, %f26; +mov.f32 %f31, %f26; +mov.f32 %f32, %f26; +mov.f32 %f33, %f26; +@!%p5 bra BB0_2; +bra.uni BB0_1; + +BB0_1: +mul.wide.s32 %rd4, %r2, 2; +add.s64 %rd5, %rd1, %rd4; +wmma.load.a.sync.row.m16n16k16.f16 {%r18, %r19, %r20, %r21, %r22, %r23, %r24, %r25}, [%rd5], %r4; +mul.wide.s32 %rd6, %r3, 2; +add.s64 %rd7, %rd2, %rd6; +wmma.load.b.sync.col.m16n16k16.f16 {%r26, %r27, %r28, %r29, %r30, %r31, %r32, %r33}, [%rd7], %r5; +mov.f32 %f25, 0f00000000; +wmma.mma.sync.row.col.m16n16k16.f32.f32 {%f33, %f32, %f31, %f30, %f29, %f28, %f27, %f26}, {%r18, %r19, %r20, %r21, %r22, %r23, %r24, %r25}, {%r26, %r27, %r28, %r29, %r30, %r31, %r32, %r33}, {%f25, %f25, %f25, %f25, %f25, %f25, %f25, %f25}; + +BB0_2: +add.u64 %rd8, %SP, 0; +cvta.to.local.u64 %rd9, %rd8; +mul.lo.s32 %r35, %r3, %r4; +cvt.s64.s32 %rd10, %r35; +cvt.s64.s32 %rd11, %r2; +add.s64 %rd12, %rd10, %rd11; +shl.b64 %rd13, %rd12, 2; +add.s64 %rd14, %rd3, %rd13; +wmma.store.d.sync.col.m16n16k16.f32 [%rd14], {%f33, %f32, %f31, %f30, %f29, %f28, %f27, %f26}, %r4; + + mov.u32 %r34, %clock; + + sub.s32 %r36, %r34, %r6; +st.local.u32 [%rd9], %r36; +mov.u64 %rd15, $str; +cvta.global.u64 %rd16, %rd15; + + { +.reg .b32 temp_param_reg; + + .param .b64 param0; +st.param.b64 [param0+0], %rd16; +.param .b64 param1; +st.param.b64 [param1+0], %rd8; +.param .b32 retval0; +call.uni (retval0), +vprintf, +( +param0, +param1 +); +ld.param.b32 %r37, [retval0+0]; + + + } + ret; +} + + +.visible .entry _Z17convertFp32ToFp16P6__halfPfi( +.param .u64 _Z17convertFp32ToFp16P6__halfPfi_param_0, +.param .u64 _Z17convertFp32ToFp16P6__halfPfi_param_1, +.param .u32 _Z17convertFp32ToFp16P6__halfPfi_param_2 +) +{ +.reg .pred %p<2>; +.reg .b16 %rs<2>; +.reg .f32 %f<2>; +.reg .b32 %r<6>; +.reg .b64 %rd<9>; + + +ld.param.u64 %rd1, [_Z17convertFp32ToFp16P6__halfPfi_param_0]; +ld.param.u64 %rd2, [_Z17convertFp32ToFp16P6__halfPfi_param_1]; +ld.param.u32 %r2, [_Z17convertFp32ToFp16P6__halfPfi_param_2]; +mov.u32 %r3, %ntid.x; +mov.u32 %r4, %ctaid.x; +mov.u32 %r5, %tid.x; +mad.lo.s32 %r1, %r4, %r3, %r5; +setp.ge.s32 %p1, %r1, %r2; +@%p1 bra BB1_2; + +cvta.to.global.u64 %rd3, %rd2; +mul.wide.s32 %rd4, %r1, 4; +add.s64 %rd5, %rd3, %rd4; +ld.global.f32 %f1, [%rd5]; + + { cvt.rn.f16.f32 %rs1, %f1;} + + + cvta.to.global.u64 %rd6, %rd1; +mul.wide.s32 %rd7, %r1, 2; +add.s64 %rd8, %rd6, %rd7; +st.global.u16 [%rd8], %rs1; + +BB1_2: +ret; +} + + diff --git a/cuda-kernels/_cuobjdump_1.sass b/cuda-kernels/_cuobjdump_1.sass new file mode 100644 index 0000000..2aac29a --- /dev/null +++ b/cuda-kernels/_cuobjdump_1.sass @@ -0,0 +1,2 @@ + code for sm_70 + diff --git a/cuda-kernels/_cuobjdump_2.elf b/cuda-kernels/_cuobjdump_2.elf new file mode 100644 index 0000000..c03b06d --- /dev/null +++ b/cuda-kernels/_cuobjdump_2.elf @@ -0,0 +1,494 @@ +64bit elf: type=2, abi=7, sm=70, toolkit=90, flags = 0x460546 +Sections: +Index Offset Size ES Align Type Flags Link Info Name + 1 40 21b 0 1 STRTAB 0 0 0 .shstrtab + 2 25b 273 0 1 STRTAB 0 0 0 .strtab + 3 4d0 108 18 8 SYMTAB 0 2 7 .symtab + 4 5d8 e0 0 1 PROGBITS 0 0 0 .debug_frame + 5 6b8 48 0 4 CUDA_INFO 0 3 0 .nv.info + 6 700 50 0 4 CUDA_INFO 0 3 d .nv.info._Z17convertFp32ToFp16P6__halfPfi + 7 750 ac 0 4 CUDA_INFO 0 3 e .nv.info._Z12wmma_exampleP6__halfS0_Pfiiiff + 8 800 30 10 8 REL 0 3 e .rel.text._Z12wmma_exampleP6__halfS0_Pfiiiff + 9 830 30 18 8 RELA 0 3 e .rela.text._Z12wmma_exampleP6__halfS0_Pfiiiff + a 860 20 10 8 REL 0 3 4 .rel.debug_frame + b 880 174 0 4 PROGBITS 2 0 d .nv.constant0._Z17convertFp32ToFp16P6__halfPfi + c 9f4 18c 0 4 PROGBITS 2 0 e .nv.constant0._Z12wmma_exampleP6__halfS0_Pfiiiff + d b80 100 0 80 PROGBITS 6 3 9000008 .text._Z17convertFp32ToFp16P6__halfPfi + e c80 980 0 80 PROGBITS 6 3 20000009 .text._Z12wmma_exampleP6__halfS0_Pfiiiff + f 1600 9 0 10 PROGBITS 3 0 0 .nv.global.init + +.section .strtab + +.section .shstrtab + +.section .symtab + index value size info other shndx name + 0 0 0 0 0 0 (null) + 1 0 0 3 0 d .text._Z17convertFp32ToFp16P6__halfPfi + 2 0 0 3 0 f .nv.global.init + 3 0 9 1 0 f $str + 4 0 0 3 0 b .nv.constant0._Z17convertFp32ToFp16P6__halfPfi + 5 0 0 3 0 e .text._Z12wmma_exampleP6__halfS0_Pfiiiff + 6 0 0 3 0 c .nv.constant0._Z12wmma_exampleP6__halfS0_Pfiiiff + 7 0 0 3 0 4 .debug_frame + 8 0 256 12 10 d _Z17convertFp32ToFp16P6__halfPfi + 9 0 2432 12 10 e _Z12wmma_exampleP6__halfS0_Pfiiiff + 10 0 0 12 0 0 vprintf + + +.nv.constant0._Z17convertFp32ToFp16P6__halfPfi +0x00000000 0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 + + + +.nv.constant0._Z12wmma_exampleP6__halfS0_Pfiiiff +0x00000000 0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 + + +.nv.global.init +0x636f6c63 0x64253d6b 0 + + +.nv.info + <0x1> + Attribute: EIATTR_MAX_STACK_SIZE + Format: EIFMT_SVAL + Value: 0x9 0x0 + <0x2> + Attribute: EIATTR_MIN_STACK_SIZE + Format: EIFMT_SVAL + Value: function: _Z12wmma_exampleP6__halfS0_Pfiiiff(0x9) min stack size: 0x8 + <0x3> + Attribute: EIATTR_FRAME_SIZE + Format: EIFMT_SVAL + Value: function: _Z12wmma_exampleP6__halfS0_Pfiiiff(0x9) frame size: 0x8 + <0x4> + Attribute: EIATTR_MAX_STACK_SIZE + Format: EIFMT_SVAL + Value: 0x8 0x0 + <0x5> + Attribute: EIATTR_MIN_STACK_SIZE + Format: EIFMT_SVAL + Value: function: _Z17convertFp32ToFp16P6__halfPfi(0x8) min stack size: 0x0 + <0x6> + Attribute: EIATTR_FRAME_SIZE + Format: EIFMT_SVAL + Value: function: _Z17convertFp32ToFp16P6__halfPfi(0x8) frame size: 0x0 + + +.nv.info._Z17convertFp32ToFp16P6__halfPfi + <0x1> + Attribute: EIATTR_PARAM_CBANK + Format: EIFMT_SVAL + Value: 0x4 0x140160 + <0x2> + Attribute: EIATTR_CBANK_PARAM_SIZE + Format: EIFMT_HVAL + Value: 0x14 + <0x3> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x2 Offset : 0x10 Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x4> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x1 Offset : 0x8 Size : 0x8 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x5> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x0 Offset : 0x0 Size : 0x8 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x6> + Attribute: EIATTR_MAXREG_COUNT + Format: EIFMT_HVAL + Value: 0xff + <0x7> + Attribute: EIATTR_EXIT_INSTR_OFFSETS + Format: EIFMT_SVAL + Value: 0x60 0xe0 + + +.nv.info._Z12wmma_exampleP6__halfS0_Pfiiiff + <0x1> + Attribute: EIATTR_PARAM_CBANK + Format: EIFMT_SVAL + Value: 0x6 0x2c0160 + <0x2> + Attribute: EIATTR_CBANK_PARAM_SIZE + Format: EIFMT_HVAL + Value: 0x2c + <0x3> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x7 Offset : 0x28 Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x4> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x6 Offset : 0x24 Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x5> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x5 Offset : 0x20 Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x6> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x4 Offset : 0x1c Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x7> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x3 Offset : 0x18 Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x8> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x2 Offset : 0x10 Size : 0x8 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x9> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x1 Offset : 0x8 Size : 0x8 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x10> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x0 Offset : 0x0 Size : 0x8 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x11> + Attribute: EIATTR_MAXREG_COUNT + Format: EIFMT_HVAL + Value: 0xff + <0x12> + Attribute: EIATTR_EXIT_INSTR_OFFSETS + Format: EIFMT_SVAL + Value: 0x940 + <0x13> + Attribute: EIATTR_EXTERNS + Format: EIFMT_SVAL + Value: externs: vprintf(0xa) + <0x14> + Attribute: EIATTR_CRS_STACK_SIZE + Format: EIFMT_SVAL + Value: 0x0 + + +.text._Z17convertFp32ToFp16P6__halfPfi +bar = 0 reg = 9 lmem=0 smem=0 +0xfffff389 0x000000ff 0x000e00ff 0x000fe200 +0x00017a02 0x00000a00 0x00000f00 0x000fd000 +0x00047919 0x00000000 0x00002500 0x000e2200 +0x00027919 0x00000000 0x00002100 0x000e2400 +0x04047a24 0x00000000 0x078e0202 0x001fca00 +0x04007a0c 0x00005c00 0x03f062f0 0x000fd800 +0x0000094d 0x00000000 0x03800000 0x000fea00 +0x00027802 0x00000004 0x00000f00 0x000fca00 +0x04027625 0x00005a00 0x078e0202 0x000fd400 +0x02027381 0x00000000 0x001ee900 0x000e2200 +0x00057802 0x00000002 0x00000f00 0x000fca00 +0x04047625 0x00005800 0x078e0205 0x000fe200 +0x00067304 0x00000002 0x00200800 0x001e3200 +0x04007386 0x00000006 0x0010e500 0x0011e200 +0x0000794d 0x00000000 0x03800000 0x000fea00 +0x00007947 0xfffffff0 0x0383ffff 0x000fc000 + + + +.text._Z12wmma_exampleP6__halfS0_Pfiiiff +bar = 0 reg = 32 lmem=0 smem=0 +0xfffff389 0x000000ff 0x000e00ff 0x000fe200 +0xff017624 0x00000a00 0x078e00ff 0x000fd000 +0x01017810 0xfffffff8 0x07ffe0ff 0x000fc800 +0x01027a10 0x00000800 0x07f1e0ff 0x000fca00 +0xff007624 0x00000900 0x000e06ff 0x000fd000 +0x00037805 0x00000000 0x00005000 0x000fd000 +0x00077906 0x00000020 0x00209000 0x000e2400 +0x00077308 0x00000007 0x00001000 0x001e2200 +0x00067919 0x00000000 0x00002500 0x000e6200 +0x00097919 0x00000000 0x00002100 0x000e6200 +0x07087810 0x0ffffffe 0x07ffe0ff 0x001fcc00 +0x00057305 0x00000008 0x0021f000 0x0000a200 +0xff047224 0x000000ff 0x078e00ff 0x000fe400 +0x06067a24 0x00000000 0x078e0209 0x002fe400 +0x050a7824 0xffffffe0 0x078e00ff 0x004fc800 +0x05047225 0x0000000a 0x078e0004 0x000fd000 +0x05047225 0x00000006 0x078e00ff 0x000fcc00 +0xff047224 0x000000ff 0x078e0a05 0x000fc800 +0x04067824 0x00000020 0x078e0206 0x000fca00 +0x0600780c 0x00000020 0x03f060f0 0x040fe200 +0x001c7919 0x00000000 0x00002600 0x000e2200 +0x00077919 0x00000000 0x00002200 0x000e3400 +0x06060810 0xffffffe0 0x07ffe0ff 0x000fc800 +0x0600780c 0x00000020 0x03f260f0 0x000fe400 +0x05050810 0x00000001 0x07ffe0ff 0x000fe400 +0xff007a0c 0x00006000 0x03f012f0 0x000fd000 +0x05051810 0x00000001 0x07ffe0ff 0x000fe200 +0x1c1c7a24 0x00000100 0x078e0207 0x001fc600 +0x051d7819 0x00000004 0x000006ff 0x000fe200 +0x1c1c7824 0x00000010 0x078e00ff 0x000fc600 +0x1d007a0c 0x00005e00 0x007012f0 0x000fc800 +0x1c007a0c 0x00005f00 0x007012f0 0x000fe200 +0x00007945 0x000003a0 0x03800000 0x000fe200 +0xff077224 0x000000ff 0x078e00ff 0x000fe200 +0x000b7202 0x000000ff 0x00000f00 0x000fe200 +0xff067224 0x000000ff 0x078e00ff 0x000fe400 +0xff057224 0x000000ff 0x078e00ff 0x000fe400 +0xff047224 0x000000ff 0x078e00ff 0x000fe400 +0xff0a7224 0x000000ff 0x078e00ff 0x000fc400 +0xff097224 0x000000ff 0x078e00ff 0x000fe400 +0xff087224 0x000000ff 0x078e00ff 0x000fe200 +0x00008947 0x00000300 0x03800000 0x000fee00 +0x00067919 0x00000000 0x00000000 0x000e2200 +0xff0a7424 0x00000002 0x078e00ff 0x000fc800 +0x1d107625 0x00005800 0x078e020a 0x000fe200 +0xff047819 0x00000002 0x00011606 0x001fc800 +0x04057812 0x00000003 0x078ec0ff 0x000fe400 +0x06047812 0x00000003 0x078ec0ff 0x000fe400 +0x05077812 0x00000001 0x078ec0ff 0x000fe400 +0xff067819 0x00000004 0x00011606 0x000fe400 +0xff057819 0x00000001 0x00011605 0x000fe200 +0x07077824 0x00000008 0x078e0204 0x000fe200 +0x06067812 0x00000001 0x078ec0ff 0x000fc400 +0x05047211 0x00000004 0x078e18ff 0x000fe200 +0x1c0c7625 0x00005a00 0x078e020a 0x000fe400 +0x06077824 0x00000004 0x078e0207 0x040fe400 +0x06047824 0x00000004 0x078e0204 0x000fe400 +0x07077824 0x00000002 0x078e00ff 0x000fe400 +0x04057824 0x00000002 0x078e00ff 0x000fe400 +0x07107a25 0x00005e00 0x078e0010 0x000fc400 +0x050c7a25 0x00006000 0x078e000c 0x000fd000 +0x10187980 0x00000000 0x0010ed00 0x00006400 +0x0c147980 0x00000000 0x0010ed00 0x00046200 +0x10107980 0x00000010 0x0010ed00 0x001e2200 +0x0c0c7980 0x00000010 0x0010ed00 0x004e2200 +0xff087224 0x000000ff 0x078e00ff 0x000fe200 +0x00097202 0x000000ff 0x00000f00 0x000fe200 +0xff0a7224 0x000000ff 0x078e00ff 0x000fe400 +0xff0b7224 0x000000ff 0x078e00ff 0x000fe200 +0x00077202 0x000000ff 0x00000f00 0x000fe200 +0xff047224 0x000000ff 0x078e00ff 0x000fc400 +0xff057224 0x000000ff 0x078e00ff 0x000fe400 +0xff067224 0x000000ff 0x078e00ff 0x000fe200 +0x00007948 0xffffffff 0x03800000 0x000fe200 +0x18087236 0x00000014 0x00005408 0x0c226400 +0x180a7236 0x00000014 0x0000d40a 0x0c04a400 +0x18047236 0x00000014 0x00015404 0x0c06e400 +0x18067236 0x00000014 0x0001d406 0x00092800 +0x1a087236 0x00000016 0x00005408 0x0c202400 +0x1a0a7236 0x00000016 0x0000d40a 0x0c426400 +0x1a047236 0x00000016 0x00015404 0x0c84a400 +0x1a067236 0x00000016 0x0001d406 0x0106e800 +0x10087236 0x0000000c 0x00005408 0x0c102400 +0x100a7236 0x0000000c 0x0000d40a 0x0c226400 +0x10047236 0x0000000c 0x00015404 0x0c44a400 +0x10067236 0x0000000c 0x0001d406 0x0086e800 +0x12087236 0x0000000e 0x00005408 0x0c102400 +0x120a7236 0x0000000e 0x0000d40a 0x0c202400 +0x12047236 0x0000000e 0x00015404 0x0c402400 +0x12067236 0x0000000e 0x0001d406 0x00803400 +0x00007941 0x00000000 0x03800000 0x001fea00 +0x000c7919 0x00000000 0x00000000 0x000e2200 +0x1c1c7a24 0x00005e00 0x078e02ff 0x000fe200 +0xff0e7819 0x00000004 0x0001160c 0x001fc400 +0xff0d7819 0x00000002 0x0001160c 0x000fe400 +0x0c0c7812 0x00000003 0x078ec0ff 0x000fe400 +0x0e0e7812 0x00000001 0x078ec0ff 0x000fe400 +0x0d0d7812 0x00000003 0x078ec0ff 0x000fc600 +0x0e0c7824 0x00000004 0x078e020c 0x000fe200 +0x0d0f7812 0x00000001 0x078ec0ff 0x000fe400 +0xff107819 0x00000001 0x0001160d 0x000fe400 +0x0c0d7812 0x00000005 0x078ec0ff 0x040fe400 +0x0c0e7812 0x00000002 0x078ec0ff 0x000fc600 +0x0f0c7824 0x00000008 0x078e020d 0x000fe200 +0xff0f7819 0x0000001f 0x0001141d 0x000fe200 +0x100e7824 0x00000008 0x078e020e 0x000fe200 +0x1d117210 0x0000001c 0x07f1e0ff 0x000fe200 +0xff0d7224 0x000000ff 0x078e00ff 0x000fc600 +0x1c0f7211 0x0000000f 0x000f0eff 0x000fe200 +0x0e0c7a25 0x00005e00 0x078e000c 0x000fe200 +0x110e7a11 0x00005c00 0x078010ff 0x000fe200 +0xff127624 0x00005e00 0x078e00ff 0x000fc600 +0x11117a11 0x00005d00 0x000f140f 0x000fe400 +0x0c107211 0x0000000e 0x078010ff 0x000fe400 +0x120e7819 0x00000002 0x000006ff 0x000fe400 +0xff0f7819 0x0000001e 0x00011612 0x000fe400 +0x0c0d7211 0x00000011 0x000f140d 0x000fe400 +0x0e127211 0x00000010 0x078210ff 0x000fc400 +0x0e117210 0x00000010 0x07f1e0ff 0x040fe400 +0x0e137211 0x0000000d 0x008f140f 0x040fe400 +0x0e157210 0x00000012 0x07f3e0ff 0x000fe200 +0x0f147824 0x00000001 0x000e060d 0x040fe400 +0xff0c7224 0x000000ff 0x078e0010 0x000fe400 +0x0f167824 0x00000001 0x008e0613 0x000fe400 +0xff0e7224 0x000000ff 0x078e0011 0x000fc400 +0xff0f7224 0x000000ff 0x078e0014 0x000fe200 +0x00107202 0x00000015 0x00000f00 0x000fe200 +0xff117224 0x000000ff 0x078e0016 0x000fe200 +0x0c007385 0x00000000 0x0010e908 0x0001e200 +0x0c007385 0x00000008 0x0010e90a 0x0003e800 +0x0e007385 0x00000000 0x0010e909 0x0003e200 +0x0e007385 0x00000008 0x0010e90b 0x0003e200 +0x12007385 0x00000000 0x0010e904 0x0003e200 +0x12007385 0x00000008 0x0010e906 0x0003e200 +0x10007385 0x00000000 0x0010e905 0x0003e200 +0x10007385 0x00000008 0x0010e907 0x0003e200 +0x00007948 0xffffffff 0x03800000 0x000fe200 +0x02087a10 0x80000800 0x07ffe0ff 0x001fd000 +0x00047805 0x00000000 0x00005000 0x002fd000 +0x04037824 0x00000001 0x078e0a03 0x000fd000 +0x08007387 0x00000003 0x00100800 0x0001e200 +0xff067224 0x000000ff 0x078e0002 0x000fe200 +0x00047802 0x00000000 0x00000f00 0x000fe200 +0xff077224 0x000000ff 0x078e0000 0x000fe200 +0x00057802 0x00000000 0x00000f00 0x000fe400 +0x00147802 0x00000000 0x00000f00 0x000fe400 +0x00157802 0x00000000 0x00000f00 0x000fd000 +0x00007943 0x00000000 0x03c00000 0x001fea00 +0x0000794d 0x00000000 0x03800000 0x000fea00 +0x00007947 0xfffffff0 0x0383ffff 0x000fc000 +0x00007918 0x00000000 0x00000000 0x000fc000 +0x00007918 0x00000000 0x00000000 0x000fc000 + + +.section .rel.text._Z12wmma_exampleP6__halfS0_Pfiiiff REL +2272 $str R_CUDA_ABS32_LO_32 +2304 $str R_CUDA_ABS32_HI_32 +2352 vprintf R_CUDA_ABS47_34 + +.section .rela.text._Z12wmma_exampleP6__halfS0_Pfiiiff RELA +2320 _Z12wmma_exampleP6__halfS0_Pfiiiff R_CUDA_ABS32_LO_32 2368 +2336 _Z12wmma_exampleP6__halfS0_Pfiiiff R_CUDA_ABS32_HI_32 2368 + +.section .debug_frame +decodeDebugFrame, frameBuf 0xffffffff, total_length 224 +CIE length 40, cie_id -1 +version 3 +augmentation slen 1 +augmentation +code_align_factor slen 1 +data_align_factor slen 1 + Debug Frame Common Information Entry + length: 40 + CIE_id : -1 + version: 3 + augmentation: + code align factor: 4 + data align factor: -4 + return address register 16777215 + initial instructions: 23 bytes, ptr = 0x8080810c, frameBuf = 0xffffffff + DW_CFA_def_cfa register R1, offset 0 + DW_CFA_same_value R255 + DW_CFA_same_value R1 + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + Debug Frame Description Entry + length: 48 + CIE_pointer: 0 + initial_location: 0x0 + address_range: 0x100 + instructions: 24 bytes + DW_CFA_advance_loc4 delta 4 + DW_CFA_advance_loc4 delta 0 + DW_CFA_def_cfa register R1, offset 0 + DW_CFA_advance_loc4 delta 52 + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop +CIE length 40, cie_id -1 +version 3 +augmentation slen 1 +augmentation +code_align_factor slen 1 +data_align_factor slen 1 + Debug Frame Common Information Entry + length: 40 + CIE_id : -1 + version: 3 + augmentation: + code align factor: 4 + data align factor: -4 + return address register 16777215 + initial instructions: 23 bytes, ptr = 0x8080810c, frameBuf = 0xffffffff + DW_CFA_def_cfa register R1, offset 0 + DW_CFA_same_value R255 + DW_CFA_same_value R1 + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + Debug Frame Description Entry + length: 48 + CIE_pointer: 0 + initial_location: 0x0 + address_range: 0x970 + instructions: 24 bytes + DW_CFA_advance_loc4 delta 4 + DW_CFA_advance_loc4 delta 2 + DW_CFA_def_cfa register R1, offset 8 + DW_CFA_advance_loc4 delta 586 + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + +.section .rel.debug_frame REL +72 _Z17convertFp32ToFp16P6__halfPfi R_NV_64 +184 _Z12wmma_exampleP6__halfS0_Pfiiiff R_NV_64 + diff --git a/cuda-kernels/_cuobjdump_2.sass b/cuda-kernels/_cuobjdump_2.sass new file mode 100644 index 0000000..1b50ed2 --- /dev/null +++ b/cuda-kernels/_cuobjdump_2.sass @@ -0,0 +1,348 @@ + code for sm_70 + Function : _Z17convertFp32ToFp16P6__halfPfi + .headerflags @"EF_CUDA_SM70 EF_CUDA_PTX_SM(EF_CUDA_SM70)" + /*0000*/ @!PT SHFL.IDX PT, RZ, RZ, RZ, RZ; /* 0x000000fffffff389 */ + /* 0x000fe200000e00ff */ + /*0010*/ MOV R1, c[0x0][0x28]; /* 0x00000a0000017a02 */ + /* 0x000fd00000000f00 */ + /*0020*/ S2R R4, SR_CTAID.X; /* 0x0000000000047919 */ + /* 0x000e220000002500 */ + /*0030*/ S2R R2, SR_TID.X; /* 0x0000000000027919 */ + /* 0x000e240000002100 */ + /*0040*/ IMAD R4, R4, c[0x0][0x0], R2; /* 0x0000000004047a24 */ + /* 0x001fca00078e0202 */ + /*0050*/ ISETP.GE.AND P0, PT, R4, c[0x0][0x170], PT, !PT; /* 0x00005c0004007a0c */ + /* 0x000fd80003f062f0 */ + /*0060*/ @P0 EXIT; /* 0x000000000000094d */ + /* 0x000fea0003800000 */ + /*0070*/ MOV R2, 0x4; /* 0x0000000400027802 */ + /* 0x000fca0000000f00 */ + /*0080*/ IMAD.WIDE R2, R4, R2, c[0x0][0x168]; /* 0x00005a0004027625 */ + /* 0x000fd400078e0202 */ + /*0090*/ LDG.E.SYS R2, [R2]; /* 0x0000000002027381 */ + /* 0x000e2200001ee900 */ + /*00a0*/ MOV R5, 0x2; /* 0x0000000200057802 */ + /* 0x000fca0000000f00 */ + /*00b0*/ IMAD.WIDE R4, R4, R5, c[0x0][0x160]; /* 0x0000580004047625 */ + /* 0x000fe200078e0205 */ + /*00c0*/ F2F.F16.F32 R6, R2; /* 0x0000000200067304 */ + /* 0x001e320000200800 */ + /*00d0*/ STG.E.U16.SYS [R4], R6; /* 0x0000000604007386 */ + /* 0x0011e2000010e500 */ + /*00e0*/ EXIT; /* 0x000000000000794d */ + /* 0x000fea0003800000 */ + /*00f0*/ BRA 0xf0; /* 0xfffffff000007947 */ + /* 0x000fc0000383ffff */ + ........................................... + + + Function : _Z12wmma_exampleP6__halfS0_Pfiiiff + .headerflags @"EF_CUDA_SM70 EF_CUDA_PTX_SM(EF_CUDA_SM70)" + /*0000*/ @!PT SHFL.IDX PT, RZ, RZ, RZ, RZ; /* 0x000000fffffff389 */ + /* 0x000fe200000e00ff */ + /*0010*/ IMAD.U32 R1, RZ, RZ, c[0x0][0x28]; /* 0x00000a00ff017624 */ + /* 0x000fd000078e00ff */ + /*0020*/ IADD3 R1, R1, -0x8, RZ; /* 0xfffffff801017810 */ + /* 0x000fc80007ffe0ff */ + /*0030*/ IADD3 R2, P0, R1, c[0x0][0x20], RZ; /* 0x0000080001027a10 */ + /* 0x000fca0007f1e0ff */ + /*0040*/ IMAD.X R0, RZ, RZ, c[0x0][0x24], P0; /* 0x00000900ff007624 */ + /* 0x000fd000000e06ff */ + /*0050*/ CS2R.32 R3, SR_CLOCKLO; /* 0x0000000000037805 */ + /* 0x000fd00000005000 */ + /*0060*/ I2F.U32.RP R7, 0x20; /* 0x0000002000077906 */ + /* 0x000e240000209000 */ + /*0070*/ MUFU.RCP R7, R7; /* 0x0000000700077308 */ + /* 0x001e220000001000 */ + /*0080*/ S2R R6, SR_CTAID.X; /* 0x0000000000067919 */ + /* 0x000e620000002500 */ + /*0090*/ S2R R9, SR_TID.X; /* 0x0000000000097919 */ + /* 0x000e620000002100 */ + /*00a0*/ IADD3 R8, R7, 0xffffffe, RZ; /* 0x0ffffffe07087810 */ + /* 0x001fcc0007ffe0ff */ + /*00b0*/ F2I.FTZ.U32.TRUNC.NTZ R5, R8; /* 0x0000000800057305 */ + /* 0x0000a2000021f000 */ + /*00c0*/ IMAD.U32 R4, RZ, RZ, RZ; /* 0x000000ffff047224 */ + /* 0x000fe400078e00ff */ + /*00d0*/ IMAD R6, R6, c[0x0][0x0], R9; /* 0x0000000006067a24 */ + /* 0x002fe400078e0209 */ + /*00e0*/ IMAD.U32 R10, R5, -0x20, RZ; /* 0xffffffe0050a7824 */ + /* 0x004fc800078e00ff */ + /*00f0*/ IMAD.WIDE.U32 R4, R5, R10, R4; /* 0x0000000a05047225 */ + /* 0x000fd000078e0004 */ + /*0100*/ IMAD.WIDE.U32 R4, R5, R6, RZ; /* 0x0000000605047225 */ + /* 0x000fcc00078e00ff */ + /*0110*/ IMAD R4, RZ, RZ, -R5; /* 0x000000ffff047224 */ + /* 0x000fc800078e0a05 */ + /*0120*/ IMAD R6, R4, 0x20, R6; /* 0x0000002004067824 */ + /* 0x000fca00078e0206 */ + /*0130*/ ISETP.GE.U32.AND P0, PT, R6.reuse, 0x20, PT, !PT; /* 0x000000200600780c */ + /* 0x040fe20003f060f0 */ + /*0140*/ S2R R28, SR_CTAID.Y; /* 0x00000000001c7919 */ + /* 0x000e220000002600 */ + /*0150*/ S2R R7, SR_TID.Y; /* 0x0000000000077919 */ + /* 0x000e340000002200 */ + /*0160*/ @P0 IADD3 R6, R6, -0x20, RZ; /* 0xffffffe006060810 */ + /* 0x000fc80007ffe0ff */ + /*0170*/ ISETP.GE.U32.AND P1, PT, R6, 0x20, PT, !PT; /* 0x000000200600780c */ + /* 0x000fe40003f260f0 */ + /*0180*/ @P0 IADD3 R5, R5, 0x1, RZ; /* 0x0000000105050810 */ + /* 0x000fe40007ffe0ff */ + /*0190*/ ISETP.LT.AND P0, PT, RZ, c[0x0][0x180], PT, !PT; /* 0x00006000ff007a0c */ + /* 0x000fd00003f012f0 */ + /*01a0*/ @P1 IADD3 R5, R5, 0x1, RZ; /* 0x0000000105051810 */ + /* 0x000fe20007ffe0ff */ + /*01b0*/ IMAD R28, R28, c[0x0][0x4], R7; /* 0x000001001c1c7a24 */ + /* 0x001fc600078e0207 */ + /*01c0*/ SHF.L.U32 R29, R5, 0x4, RZ; /* 0x00000004051d7819 */ + /* 0x000fe200000006ff */ + /*01d0*/ IMAD.U32 R28, R28, 0x10, RZ; /* 0x000000101c1c7824 */ + /* 0x000fc600078e00ff */ + /*01e0*/ ISETP.LT.AND P0, PT, R29, c[0x0][0x178], P0, !PT; /* 0x00005e001d007a0c */ + /* 0x000fc800007012f0 */ + /*01f0*/ ISETP.LT.AND P0, PT, R28, c[0x0][0x17c], P0, !PT; /* 0x00005f001c007a0c */ + /* 0x000fe200007012f0 */ + /*0200*/ BSSY B0, 0x5b0; /* 0x000003a000007945 */ + /* 0x000fe20003800000 */ + /*0210*/ IMAD.U32 R7, RZ, RZ, RZ; /* 0x000000ffff077224 */ + /* 0x000fe200078e00ff */ + /*0220*/ MOV R11, RZ; /* 0x000000ff000b7202 */ + /* 0x000fe20000000f00 */ + /*0230*/ IMAD.U32 R6, RZ, RZ, RZ; /* 0x000000ffff067224 */ + /* 0x000fe400078e00ff */ + /*0240*/ IMAD.U32 R5, RZ, RZ, RZ; /* 0x000000ffff057224 */ + /* 0x000fe400078e00ff */ + /*0250*/ IMAD.U32 R4, RZ, RZ, RZ; /* 0x000000ffff047224 */ + /* 0x000fe400078e00ff */ + /*0260*/ IMAD.U32 R10, RZ, RZ, RZ; /* 0x000000ffff0a7224 */ + /* 0x000fc400078e00ff */ + /*0270*/ IMAD.U32 R9, RZ, RZ, RZ; /* 0x000000ffff097224 */ + /* 0x000fe400078e00ff */ + /*0280*/ IMAD.U32 R8, RZ, RZ, RZ; /* 0x000000ffff087224 */ + /* 0x000fe200078e00ff */ + /*0290*/ @!P0 BRA 0x5a0; /* 0x0000030000008947 */ + /* 0x000fee0003800000 */ + /*02a0*/ S2R R6, SR_LANEID; /* 0x0000000000067919 */ + /* 0x000e220000000000 */ + /*02b0*/ IMAD.U32 R10, RZ, RZ, 0x2; /* 0x00000002ff0a7424 */ + /* 0x000fc800078e00ff */ + /*02c0*/ IMAD.WIDE R16, R29, R10, c[0x0][0x160]; /* 0x000058001d107625 */ + /* 0x000fe200078e020a */ + /*02d0*/ SHF.R.U32.HI R4, RZ, 0x2, R6; /* 0x00000002ff047819 */ + /* 0x001fc80000011606 */ + /*02e0*/ LOP3.LUT R5, R4, 0x3, RZ, 0xc0, !PT; /* 0x0000000304057812 */ + /* 0x000fe400078ec0ff */ + /*02f0*/ LOP3.LUT R4, R6, 0x3, RZ, 0xc0, !PT; /* 0x0000000306047812 */ + /* 0x000fe400078ec0ff */ + /*0300*/ LOP3.LUT R7, R5, 0x1, RZ, 0xc0, !PT; /* 0x0000000105077812 */ + /* 0x000fe400078ec0ff */ + /*0310*/ SHF.R.U32.HI R6, RZ, 0x4, R6; /* 0x00000004ff067819 */ + /* 0x000fe40000011606 */ + /*0320*/ SHF.R.U32.HI R5, RZ, 0x1, R5; /* 0x00000001ff057819 */ + /* 0x000fe20000011605 */ + /*0330*/ IMAD R7, R7, 0x8, R4; /* 0x0000000807077824 */ + /* 0x000fe200078e0204 */ + /*0340*/ LOP3.LUT R6, R6, 0x1, RZ, 0xc0, !PT; /* 0x0000000106067812 */ + /* 0x000fc400078ec0ff */ + /*0350*/ LEA R4, R5, R4, 0x3; /* 0x0000000405047211 */ + /* 0x000fe200078e18ff */ + /*0360*/ IMAD.WIDE R12, R28, R10, c[0x0][0x168]; /* 0x00005a001c0c7625 */ + /* 0x000fe400078e020a */ + /*0370*/ IMAD R7, R6.reuse, 0x4, R7; /* 0x0000000406077824 */ + /* 0x040fe400078e0207 */ + /*0380*/ IMAD R4, R6, 0x4, R4; /* 0x0000000406047824 */ + /* 0x000fe400078e0204 */ + /*0390*/ IMAD.U32 R7, R7, 0x2, RZ; /* 0x0000000207077824 */ + /* 0x000fe400078e00ff */ + /*03a0*/ IMAD.U32 R5, R4, 0x2, RZ; /* 0x0000000204057824 */ + /* 0x000fe400078e00ff */ + /*03b0*/ IMAD.WIDE.U32 R16, R7, c[0x0][0x178], R16; /* 0x00005e0007107a25 */ + /* 0x000fc400078e0010 */ + /*03c0*/ IMAD.WIDE.U32 R12, R5, c[0x0][0x180], R12; /* 0x00006000050c7a25 */ + /* 0x000fd000078e000c */ + /*03d0*/ LD.E.128.SYS R24, [R16]; /* 0x0000000010187980 */ + /* 0x000064000010ed00 */ + /*03e0*/ LD.E.128.SYS R20, [R12]; /* 0x000000000c147980 */ + /* 0x000462000010ed00 */ + /*03f0*/ LD.E.128.SYS R16, [R16+0x10]; /* 0x0000001010107980 */ + /* 0x001e22000010ed00 */ + /*0400*/ LD.E.128.SYS R12, [R12+0x10]; /* 0x000000100c0c7980 */ + /* 0x004e22000010ed00 */ + /*0410*/ IMAD.U32 R8, RZ, RZ, RZ; /* 0x000000ffff087224 */ + /* 0x000fe200078e00ff */ + /*0420*/ MOV R9, RZ; /* 0x000000ff00097202 */ + /* 0x000fe20000000f00 */ + /*0430*/ IMAD.U32 R10, RZ, RZ, RZ; /* 0x000000ffff0a7224 */ + /* 0x000fe400078e00ff */ + /*0440*/ IMAD.U32 R11, RZ, RZ, RZ; /* 0x000000ffff0b7224 */ + /* 0x000fe200078e00ff */ + /*0450*/ MOV R7, RZ; /* 0x000000ff00077202 */ + /* 0x000fe20000000f00 */ + /*0460*/ IMAD.U32 R4, RZ, RZ, RZ; /* 0x000000ffff047224 */ + /* 0x000fc400078e00ff */ + /*0470*/ IMAD.U32 R5, RZ, RZ, RZ; /* 0x000000ffff057224 */ + /* 0x000fe400078e00ff */ + /*0480*/ IMAD.U32 R6, RZ, RZ, RZ; /* 0x000000ffff067224 */ + /* 0x000fe200078e00ff */ + /*0490*/ WARPSYNC 0xffffffff; /* 0xffffffff00007948 */ + /* 0x000fe20003800000 */ + /*04a0*/ HMMA.884.F32.F32.STEP0 R8, R24.reuse, R20.reuse.T, R8; /* 0x0000001418087236 */ + /* 0x0c22640000005408 */ + /*04b0*/ HMMA.884.F32.F32.STEP1 R10, R24.reuse, R20.reuse.T, R10; /* 0x00000014180a7236 */ + /* 0x0c04a4000000d40a */ + /*04c0*/ HMMA.884.F32.F32.STEP2 R4, R24.reuse, R20.reuse.T, R4; /* 0x0000001418047236 */ + /* 0x0c06e40000015404 */ + /*04d0*/ HMMA.884.F32.F32.STEP3 R6, R24, R20.T, R6; /* 0x0000001418067236 */ + /* 0x000928000001d406 */ + /*04e0*/ HMMA.884.F32.F32.STEP0 R8, R26.reuse, R22.reuse.T, R8; /* 0x000000161a087236 */ + /* 0x0c20240000005408 */ + /*04f0*/ HMMA.884.F32.F32.STEP1 R10, R26.reuse, R22.reuse.T, R10; /* 0x000000161a0a7236 */ + /* 0x0c4264000000d40a */ + /*0500*/ HMMA.884.F32.F32.STEP2 R4, R26.reuse, R22.reuse.T, R4; /* 0x000000161a047236 */ + /* 0x0c84a40000015404 */ + /*0510*/ HMMA.884.F32.F32.STEP3 R6, R26, R22.T, R6; /* 0x000000161a067236 */ + /* 0x0106e8000001d406 */ + /*0520*/ HMMA.884.F32.F32.STEP0 R8, R16.reuse, R12.reuse.T, R8; /* 0x0000000c10087236 */ + /* 0x0c10240000005408 */ + /*0530*/ HMMA.884.F32.F32.STEP1 R10, R16.reuse, R12.reuse.T, R10; /* 0x0000000c100a7236 */ + /* 0x0c2264000000d40a */ + /*0540*/ HMMA.884.F32.F32.STEP2 R4, R16.reuse, R12.reuse.T, R4; /* 0x0000000c10047236 */ + /* 0x0c44a40000015404 */ + /*0550*/ HMMA.884.F32.F32.STEP3 R6, R16, R12.T, R6; /* 0x0000000c10067236 */ + /* 0x0086e8000001d406 */ + /*0560*/ HMMA.884.F32.F32.STEP0 R8, R18.reuse, R14.reuse.T, R8; /* 0x0000000e12087236 */ + /* 0x0c10240000005408 */ + /*0570*/ HMMA.884.F32.F32.STEP1 R10, R18.reuse, R14.reuse.T, R10; /* 0x0000000e120a7236 */ + /* 0x0c2024000000d40a */ + /*0580*/ HMMA.884.F32.F32.STEP2 R4, R18.reuse, R14.reuse.T, R4; /* 0x0000000e12047236 */ + /* 0x0c40240000015404 */ + /*0590*/ HMMA.884.F32.F32.STEP3 R6, R18, R14.T, R6; /* 0x0000000e12067236 */ + /* 0x008034000001d406 */ + /*05a0*/ BSYNC B0; /* 0x0000000000007941 */ + /* 0x001fea0003800000 */ + /*05b0*/ S2R R12, SR_LANEID; /* 0x00000000000c7919 */ + /* 0x000e220000000000 */ + /*05c0*/ IMAD R28, R28, c[0x0][0x178], RZ; /* 0x00005e001c1c7a24 */ + /* 0x000fe200078e02ff */ + /*05d0*/ SHF.R.U32.HI R14, RZ, 0x4, R12; /* 0x00000004ff0e7819 */ + /* 0x001fc4000001160c */ + /*05e0*/ SHF.R.U32.HI R13, RZ, 0x2, R12; /* 0x00000002ff0d7819 */ + /* 0x000fe4000001160c */ + /*05f0*/ LOP3.LUT R12, R12, 0x3, RZ, 0xc0, !PT; /* 0x000000030c0c7812 */ + /* 0x000fe400078ec0ff */ + /*0600*/ LOP3.LUT R14, R14, 0x1, RZ, 0xc0, !PT; /* 0x000000010e0e7812 */ + /* 0x000fe400078ec0ff */ + /*0610*/ LOP3.LUT R13, R13, 0x3, RZ, 0xc0, !PT; /* 0x000000030d0d7812 */ + /* 0x000fc600078ec0ff */ + /*0620*/ IMAD R12, R14, 0x4, R12; /* 0x000000040e0c7824 */ + /* 0x000fe200078e020c */ + /*0630*/ LOP3.LUT R15, R13, 0x1, RZ, 0xc0, !PT; /* 0x000000010d0f7812 */ + /* 0x000fe400078ec0ff */ + /*0640*/ SHF.R.U32.HI R16, RZ, 0x1, R13; /* 0x00000001ff107819 */ + /* 0x000fe4000001160d */ + /*0650*/ LOP3.LUT R13, R12.reuse, 0x5, RZ, 0xc0, !PT; /* 0x000000050c0d7812 */ + /* 0x040fe400078ec0ff */ + /*0660*/ LOP3.LUT R14, R12, 0x2, RZ, 0xc0, !PT; /* 0x000000020c0e7812 */ + /* 0x000fc600078ec0ff */ + /*0670*/ IMAD R12, R15, 0x8, R13; /* 0x000000080f0c7824 */ + /* 0x000fe200078e020d */ + /*0680*/ SHF.R.S32.HI R15, RZ, 0x1f, R29; /* 0x0000001fff0f7819 */ + /* 0x000fe2000001141d */ + /*0690*/ IMAD R14, R16, 0x8, R14; /* 0x00000008100e7824 */ + /* 0x000fe200078e020e */ + /*06a0*/ IADD3 R17, P0, R29, R28, RZ; /* 0x0000001c1d117210 */ + /* 0x000fe20007f1e0ff */ + /*06b0*/ IMAD.U32 R13, RZ, RZ, RZ; /* 0x000000ffff0d7224 */ + /* 0x000fc600078e00ff */ + /*06c0*/ LEA.HI.X.SX32 R15, R28, R15, 0x1, P0; /* 0x0000000f1c0f7211 */ + /* 0x000fe200000f0eff */ + /*06d0*/ IMAD.WIDE.U32 R12, R14, c[0x0][0x178], R12; /* 0x00005e000e0c7a25 */ + /* 0x000fe200078e000c */ + /*06e0*/ LEA R14, P0, R17, c[0x0][0x170], 0x2; /* 0x00005c00110e7a11 */ + /* 0x000fe200078010ff */ + /*06f0*/ IMAD.U32 R18, RZ, RZ, c[0x0][0x178]; /* 0x00005e00ff127624 */ + /* 0x000fc600078e00ff */ + /*0700*/ LEA.HI.X R17, R17, c[0x0][0x174], R15, 0x2, P0; /* 0x00005d0011117a11 */ + /* 0x000fe400000f140f */ + /*0710*/ LEA R16, P0, R12, R14, 0x2; /* 0x0000000e0c107211 */ + /* 0x000fe400078010ff */ + /*0720*/ SHF.L.U32 R14, R18, 0x2, RZ; /* 0x00000002120e7819 */ + /* 0x000fe400000006ff */ + /*0730*/ SHF.R.U32.HI R15, RZ, 0x1e, R18; /* 0x0000001eff0f7819 */ + /* 0x000fe40000011612 */ + /*0740*/ LEA.HI.X R13, R12, R17, R13, 0x2, P0; /* 0x000000110c0d7211 */ + /* 0x000fe400000f140d */ + /*0750*/ LEA R18, P1, R14, R16, 0x2; /* 0x000000100e127211 */ + /* 0x000fc400078210ff */ + /*0760*/ IADD3 R17, P0, R14.reuse, R16, RZ; /* 0x000000100e117210 */ + /* 0x040fe40007f1e0ff */ + /*0770*/ LEA.HI.X R19, R14.reuse, R13, R15, 0x2, P1; /* 0x0000000d0e137211 */ + /* 0x040fe400008f140f */ + /*0780*/ IADD3 R21, P1, R14, R18, RZ; /* 0x000000120e157210 */ + /* 0x000fe20007f3e0ff */ + /*0790*/ IMAD.X R20, R15.reuse, 0x1, R13, P0; /* 0x000000010f147824 */ + /* 0x040fe400000e060d */ + /*07a0*/ IMAD.U32 R12, RZ, RZ, R16; /* 0x000000ffff0c7224 */ + /* 0x000fe400078e0010 */ + /*07b0*/ IMAD.X R22, R15, 0x1, R19, P1; /* 0x000000010f167824 */ + /* 0x000fe400008e0613 */ + /*07c0*/ IMAD.U32 R14, RZ, RZ, R17; /* 0x000000ffff0e7224 */ + /* 0x000fc400078e0011 */ + /*07d0*/ IMAD.U32 R15, RZ, RZ, R20; /* 0x000000ffff0f7224 */ + /* 0x000fe200078e0014 */ + /*07e0*/ MOV R16, R21; /* 0x0000001500107202 */ + /* 0x000fe20000000f00 */ + /*07f0*/ IMAD.U32 R17, RZ, RZ, R22; /* 0x000000ffff117224 */ + /* 0x000fe200078e0016 */ + /*0800*/ ST.E.SYS [R12], R8; /* 0x000000000c007385 */ + /* 0x0001e2000010e908 */ + /*0810*/ ST.E.SYS [R12+0x8], R10; /* 0x000000080c007385 */ + /* 0x0003e8000010e90a */ + /*0820*/ ST.E.SYS [R14], R9; /* 0x000000000e007385 */ + /* 0x0003e2000010e909 */ + /*0830*/ ST.E.SYS [R14+0x8], R11; /* 0x000000080e007385 */ + /* 0x0003e2000010e90b */ + /*0840*/ ST.E.SYS [R18], R4; /* 0x0000000012007385 */ + /* 0x0003e2000010e904 */ + /*0850*/ ST.E.SYS [R18+0x8], R6; /* 0x0000000812007385 */ + /* 0x0003e2000010e906 */ + /*0860*/ ST.E.SYS [R16], R5; /* 0x0000000010007385 */ + /* 0x0003e2000010e905 */ + /*0870*/ ST.E.SYS [R16+0x8], R7; /* 0x0000000810007385 */ + /* 0x0003e2000010e907 */ + /*0880*/ WARPSYNC 0xffffffff; /* 0xffffffff00007948 */ + /* 0x000fe20003800000 */ + /*0890*/ IADD3 R8, R2, -c[0x0][0x20], RZ; /* 0x8000080002087a10 */ + /* 0x001fd00007ffe0ff */ + /*08a0*/ CS2R.32 R4, SR_CLOCKLO; /* 0x0000000000047805 */ + /* 0x002fd00000005000 */ + /*08b0*/ IMAD R3, R4, 0x1, -R3; /* 0x0000000104037824 */ + /* 0x000fd000078e0a03 */ + /*08c0*/ STL [R8], R3; /* 0x0000000308007387 */ + /* 0x0001e20000100800 */ + /*08d0*/ IMAD.U32 R6, RZ, RZ, R2; /* 0x000000ffff067224 */ + /* 0x000fe200078e0002 */ + /*08e0*/ MOV R4, 0x0; /* 0x0000000000047802 */ + /* 0x000fe20000000f00 */ + /*08f0*/ IMAD.U32 R7, RZ, RZ, R0; /* 0x000000ffff077224 */ + /* 0x000fe200078e0000 */ + /*0900*/ MOV R5, 0x0; /* 0x0000000000057802 */ + /* 0x000fe40000000f00 */ + /*0910*/ MOV R20, 0x0; /* 0x0000000000147802 */ + /* 0x000fe40000000f00 */ + /*0920*/ MOV R21, 0x0; /* 0x0000000000157802 */ + /* 0x000fd00000000f00 */ + /*0930*/ CALL.ABS.NOINC 0x0; /* 0x0000000000007943 */ + /* 0x001fea0003c00000 */ + /*0940*/ EXIT; /* 0x000000000000794d */ + /* 0x000fea0003800000 */ + /*0950*/ BRA 0x950; /* 0xfffffff000007947 */ + /* 0x000fc0000383ffff */ + /*0960*/ NOP; /* 0x0000000000007918 */ + /* 0x000fc00000000000 */ + /*0970*/ NOP; /* 0x0000000000007918 */ + /* 0x000fc00000000000 */ + ............................................. + + + diff --git a/cuda-kernels/_cuobjdump_complete_output_EIGzTK b/cuda-kernels/_cuobjdump_complete_output_EIGzTK new file mode 100644 index 0000000..36999c0 --- /dev/null +++ b/cuda-kernels/_cuobjdump_complete_output_EIGzTK @@ -0,0 +1,1055 @@ + +Fatbin elf code: +================ +arch = sm_70 +code version = [1,7] +producer = +host = linux +compile_size = 64bit + +64bit elf: type=2, abi=7, sm=70, toolkit=90, flags = 0x460546 +Sections: +Index Offset Size ES Align Type Flags Link Info Name + 1 40 32 0 1 STRTAB 0 0 0 .shstrtab + 2 72 32 0 1 STRTAB 0 0 0 .strtab + 3 a8 18 18 8 SYMTAB 0 2 0 .symtab + +.section .strtab + +.section .shstrtab + +.section .symtab + index value size info other shndx name + 0 0 0 0 0 0 (null) + + code for sm_70 + +Fatbin elf code: +================ +arch = sm_70 +code version = [1,7] +producer = cuda +host = linux +compile_size = 64bit + +64bit elf: type=2, abi=7, sm=70, toolkit=90, flags = 0x460546 +Sections: +Index Offset Size ES Align Type Flags Link Info Name + 1 40 21b 0 1 STRTAB 0 0 0 .shstrtab + 2 25b 273 0 1 STRTAB 0 0 0 .strtab + 3 4d0 108 18 8 SYMTAB 0 2 7 .symtab + 4 5d8 e0 0 1 PROGBITS 0 0 0 .debug_frame + 5 6b8 48 0 4 CUDA_INFO 0 3 0 .nv.info + 6 700 50 0 4 CUDA_INFO 0 3 d .nv.info._Z17convertFp32ToFp16P6__halfPfi + 7 750 ac 0 4 CUDA_INFO 0 3 e .nv.info._Z12wmma_exampleP6__halfS0_Pfiiiff + 8 800 30 10 8 REL 0 3 e .rel.text._Z12wmma_exampleP6__halfS0_Pfiiiff + 9 830 30 18 8 RELA 0 3 e .rela.text._Z12wmma_exampleP6__halfS0_Pfiiiff + a 860 20 10 8 REL 0 3 4 .rel.debug_frame + b 880 174 0 4 PROGBITS 2 0 d .nv.constant0._Z17convertFp32ToFp16P6__halfPfi + c 9f4 18c 0 4 PROGBITS 2 0 e .nv.constant0._Z12wmma_exampleP6__halfS0_Pfiiiff + d b80 100 0 80 PROGBITS 6 3 9000008 .text._Z17convertFp32ToFp16P6__halfPfi + e c80 980 0 80 PROGBITS 6 3 20000009 .text._Z12wmma_exampleP6__halfS0_Pfiiiff + f 1600 9 0 10 PROGBITS 3 0 0 .nv.global.init + +.section .strtab + +.section .shstrtab + +.section .symtab + index value size info other shndx name + 0 0 0 0 0 0 (null) + 1 0 0 3 0 d .text._Z17convertFp32ToFp16P6__halfPfi + 2 0 0 3 0 f .nv.global.init + 3 0 9 1 0 f $str + 4 0 0 3 0 b .nv.constant0._Z17convertFp32ToFp16P6__halfPfi + 5 0 0 3 0 e .text._Z12wmma_exampleP6__halfS0_Pfiiiff + 6 0 0 3 0 c .nv.constant0._Z12wmma_exampleP6__halfS0_Pfiiiff + 7 0 0 3 0 4 .debug_frame + 8 0 256 12 10 d _Z17convertFp32ToFp16P6__halfPfi + 9 0 2432 12 10 e _Z12wmma_exampleP6__halfS0_Pfiiiff + 10 0 0 12 0 0 vprintf + + +.nv.constant0._Z17convertFp32ToFp16P6__halfPfi +0x00000000 0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 + + + +.nv.constant0._Z12wmma_exampleP6__halfS0_Pfiiiff +0x00000000 0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 + + +.nv.global.init +0x636f6c63 0x64253d6b 0 + + +.nv.info + <0x1> + Attribute: EIATTR_MAX_STACK_SIZE + Format: EIFMT_SVAL + Value: 0x9 0x0 + <0x2> + Attribute: EIATTR_MIN_STACK_SIZE + Format: EIFMT_SVAL + Value: function: _Z12wmma_exampleP6__halfS0_Pfiiiff(0x9) min stack size: 0x8 + <0x3> + Attribute: EIATTR_FRAME_SIZE + Format: EIFMT_SVAL + Value: function: _Z12wmma_exampleP6__halfS0_Pfiiiff(0x9) frame size: 0x8 + <0x4> + Attribute: EIATTR_MAX_STACK_SIZE + Format: EIFMT_SVAL + Value: 0x8 0x0 + <0x5> + Attribute: EIATTR_MIN_STACK_SIZE + Format: EIFMT_SVAL + Value: function: _Z17convertFp32ToFp16P6__halfPfi(0x8) min stack size: 0x0 + <0x6> + Attribute: EIATTR_FRAME_SIZE + Format: EIFMT_SVAL + Value: function: _Z17convertFp32ToFp16P6__halfPfi(0x8) frame size: 0x0 + + +.nv.info._Z17convertFp32ToFp16P6__halfPfi + <0x1> + Attribute: EIATTR_PARAM_CBANK + Format: EIFMT_SVAL + Value: 0x4 0x140160 + <0x2> + Attribute: EIATTR_CBANK_PARAM_SIZE + Format: EIFMT_HVAL + Value: 0x14 + <0x3> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x2 Offset : 0x10 Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x4> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x1 Offset : 0x8 Size : 0x8 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x5> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x0 Offset : 0x0 Size : 0x8 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x6> + Attribute: EIATTR_MAXREG_COUNT + Format: EIFMT_HVAL + Value: 0xff + <0x7> + Attribute: EIATTR_EXIT_INSTR_OFFSETS + Format: EIFMT_SVAL + Value: 0x60 0xe0 + + +.nv.info._Z12wmma_exampleP6__halfS0_Pfiiiff + <0x1> + Attribute: EIATTR_PARAM_CBANK + Format: EIFMT_SVAL + Value: 0x6 0x2c0160 + <0x2> + Attribute: EIATTR_CBANK_PARAM_SIZE + Format: EIFMT_HVAL + Value: 0x2c + <0x3> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x7 Offset : 0x28 Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x4> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x6 Offset : 0x24 Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x5> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x5 Offset : 0x20 Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x6> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x4 Offset : 0x1c Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x7> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x3 Offset : 0x18 Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x8> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x2 Offset : 0x10 Size : 0x8 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x9> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x1 Offset : 0x8 Size : 0x8 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x10> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x0 Offset : 0x0 Size : 0x8 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x11> + Attribute: EIATTR_MAXREG_COUNT + Format: EIFMT_HVAL + Value: 0xff + <0x12> + Attribute: EIATTR_EXIT_INSTR_OFFSETS + Format: EIFMT_SVAL + Value: 0x940 + <0x13> + Attribute: EIATTR_EXTERNS + Format: EIFMT_SVAL + Value: externs: vprintf(0xa) + <0x14> + Attribute: EIATTR_CRS_STACK_SIZE + Format: EIFMT_SVAL + Value: 0x0 + + +.text._Z17convertFp32ToFp16P6__halfPfi +bar = 0 reg = 9 lmem=0 smem=0 +0xfffff389 0x000000ff 0x000e00ff 0x000fe200 +0x00017a02 0x00000a00 0x00000f00 0x000fd000 +0x00047919 0x00000000 0x00002500 0x000e2200 +0x00027919 0x00000000 0x00002100 0x000e2400 +0x04047a24 0x00000000 0x078e0202 0x001fca00 +0x04007a0c 0x00005c00 0x03f062f0 0x000fd800 +0x0000094d 0x00000000 0x03800000 0x000fea00 +0x00027802 0x00000004 0x00000f00 0x000fca00 +0x04027625 0x00005a00 0x078e0202 0x000fd400 +0x02027381 0x00000000 0x001ee900 0x000e2200 +0x00057802 0x00000002 0x00000f00 0x000fca00 +0x04047625 0x00005800 0x078e0205 0x000fe200 +0x00067304 0x00000002 0x00200800 0x001e3200 +0x04007386 0x00000006 0x0010e500 0x0011e200 +0x0000794d 0x00000000 0x03800000 0x000fea00 +0x00007947 0xfffffff0 0x0383ffff 0x000fc000 + + + +.text._Z12wmma_exampleP6__halfS0_Pfiiiff +bar = 0 reg = 32 lmem=0 smem=0 +0xfffff389 0x000000ff 0x000e00ff 0x000fe200 +0xff017624 0x00000a00 0x078e00ff 0x000fd000 +0x01017810 0xfffffff8 0x07ffe0ff 0x000fc800 +0x01027a10 0x00000800 0x07f1e0ff 0x000fca00 +0xff007624 0x00000900 0x000e06ff 0x000fd000 +0x00037805 0x00000000 0x00005000 0x000fd000 +0x00077906 0x00000020 0x00209000 0x000e2400 +0x00077308 0x00000007 0x00001000 0x001e2200 +0x00067919 0x00000000 0x00002500 0x000e6200 +0x00097919 0x00000000 0x00002100 0x000e6200 +0x07087810 0x0ffffffe 0x07ffe0ff 0x001fcc00 +0x00057305 0x00000008 0x0021f000 0x0000a200 +0xff047224 0x000000ff 0x078e00ff 0x000fe400 +0x06067a24 0x00000000 0x078e0209 0x002fe400 +0x050a7824 0xffffffe0 0x078e00ff 0x004fc800 +0x05047225 0x0000000a 0x078e0004 0x000fd000 +0x05047225 0x00000006 0x078e00ff 0x000fcc00 +0xff047224 0x000000ff 0x078e0a05 0x000fc800 +0x04067824 0x00000020 0x078e0206 0x000fca00 +0x0600780c 0x00000020 0x03f060f0 0x040fe200 +0x001c7919 0x00000000 0x00002600 0x000e2200 +0x00077919 0x00000000 0x00002200 0x000e3400 +0x06060810 0xffffffe0 0x07ffe0ff 0x000fc800 +0x0600780c 0x00000020 0x03f260f0 0x000fe400 +0x05050810 0x00000001 0x07ffe0ff 0x000fe400 +0xff007a0c 0x00006000 0x03f012f0 0x000fd000 +0x05051810 0x00000001 0x07ffe0ff 0x000fe200 +0x1c1c7a24 0x00000100 0x078e0207 0x001fc600 +0x051d7819 0x00000004 0x000006ff 0x000fe200 +0x1c1c7824 0x00000010 0x078e00ff 0x000fc600 +0x1d007a0c 0x00005e00 0x007012f0 0x000fc800 +0x1c007a0c 0x00005f00 0x007012f0 0x000fe200 +0x00007945 0x000003a0 0x03800000 0x000fe200 +0xff077224 0x000000ff 0x078e00ff 0x000fe200 +0x000b7202 0x000000ff 0x00000f00 0x000fe200 +0xff067224 0x000000ff 0x078e00ff 0x000fe400 +0xff057224 0x000000ff 0x078e00ff 0x000fe400 +0xff047224 0x000000ff 0x078e00ff 0x000fe400 +0xff0a7224 0x000000ff 0x078e00ff 0x000fc400 +0xff097224 0x000000ff 0x078e00ff 0x000fe400 +0xff087224 0x000000ff 0x078e00ff 0x000fe200 +0x00008947 0x00000300 0x03800000 0x000fee00 +0x00067919 0x00000000 0x00000000 0x000e2200 +0xff0a7424 0x00000002 0x078e00ff 0x000fc800 +0x1d107625 0x00005800 0x078e020a 0x000fe200 +0xff047819 0x00000002 0x00011606 0x001fc800 +0x04057812 0x00000003 0x078ec0ff 0x000fe400 +0x06047812 0x00000003 0x078ec0ff 0x000fe400 +0x05077812 0x00000001 0x078ec0ff 0x000fe400 +0xff067819 0x00000004 0x00011606 0x000fe400 +0xff057819 0x00000001 0x00011605 0x000fe200 +0x07077824 0x00000008 0x078e0204 0x000fe200 +0x06067812 0x00000001 0x078ec0ff 0x000fc400 +0x05047211 0x00000004 0x078e18ff 0x000fe200 +0x1c0c7625 0x00005a00 0x078e020a 0x000fe400 +0x06077824 0x00000004 0x078e0207 0x040fe400 +0x06047824 0x00000004 0x078e0204 0x000fe400 +0x07077824 0x00000002 0x078e00ff 0x000fe400 +0x04057824 0x00000002 0x078e00ff 0x000fe400 +0x07107a25 0x00005e00 0x078e0010 0x000fc400 +0x050c7a25 0x00006000 0x078e000c 0x000fd000 +0x10187980 0x00000000 0x0010ed00 0x00006400 +0x0c147980 0x00000000 0x0010ed00 0x00046200 +0x10107980 0x00000010 0x0010ed00 0x001e2200 +0x0c0c7980 0x00000010 0x0010ed00 0x004e2200 +0xff087224 0x000000ff 0x078e00ff 0x000fe200 +0x00097202 0x000000ff 0x00000f00 0x000fe200 +0xff0a7224 0x000000ff 0x078e00ff 0x000fe400 +0xff0b7224 0x000000ff 0x078e00ff 0x000fe200 +0x00077202 0x000000ff 0x00000f00 0x000fe200 +0xff047224 0x000000ff 0x078e00ff 0x000fc400 +0xff057224 0x000000ff 0x078e00ff 0x000fe400 +0xff067224 0x000000ff 0x078e00ff 0x000fe200 +0x00007948 0xffffffff 0x03800000 0x000fe200 +0x18087236 0x00000014 0x00005408 0x0c226400 +0x180a7236 0x00000014 0x0000d40a 0x0c04a400 +0x18047236 0x00000014 0x00015404 0x0c06e400 +0x18067236 0x00000014 0x0001d406 0x00092800 +0x1a087236 0x00000016 0x00005408 0x0c202400 +0x1a0a7236 0x00000016 0x0000d40a 0x0c426400 +0x1a047236 0x00000016 0x00015404 0x0c84a400 +0x1a067236 0x00000016 0x0001d406 0x0106e800 +0x10087236 0x0000000c 0x00005408 0x0c102400 +0x100a7236 0x0000000c 0x0000d40a 0x0c226400 +0x10047236 0x0000000c 0x00015404 0x0c44a400 +0x10067236 0x0000000c 0x0001d406 0x0086e800 +0x12087236 0x0000000e 0x00005408 0x0c102400 +0x120a7236 0x0000000e 0x0000d40a 0x0c202400 +0x12047236 0x0000000e 0x00015404 0x0c402400 +0x12067236 0x0000000e 0x0001d406 0x00803400 +0x00007941 0x00000000 0x03800000 0x001fea00 +0x000c7919 0x00000000 0x00000000 0x000e2200 +0x1c1c7a24 0x00005e00 0x078e02ff 0x000fe200 +0xff0e7819 0x00000004 0x0001160c 0x001fc400 +0xff0d7819 0x00000002 0x0001160c 0x000fe400 +0x0c0c7812 0x00000003 0x078ec0ff 0x000fe400 +0x0e0e7812 0x00000001 0x078ec0ff 0x000fe400 +0x0d0d7812 0x00000003 0x078ec0ff 0x000fc600 +0x0e0c7824 0x00000004 0x078e020c 0x000fe200 +0x0d0f7812 0x00000001 0x078ec0ff 0x000fe400 +0xff107819 0x00000001 0x0001160d 0x000fe400 +0x0c0d7812 0x00000005 0x078ec0ff 0x040fe400 +0x0c0e7812 0x00000002 0x078ec0ff 0x000fc600 +0x0f0c7824 0x00000008 0x078e020d 0x000fe200 +0xff0f7819 0x0000001f 0x0001141d 0x000fe200 +0x100e7824 0x00000008 0x078e020e 0x000fe200 +0x1d117210 0x0000001c 0x07f1e0ff 0x000fe200 +0xff0d7224 0x000000ff 0x078e00ff 0x000fc600 +0x1c0f7211 0x0000000f 0x000f0eff 0x000fe200 +0x0e0c7a25 0x00005e00 0x078e000c 0x000fe200 +0x110e7a11 0x00005c00 0x078010ff 0x000fe200 +0xff127624 0x00005e00 0x078e00ff 0x000fc600 +0x11117a11 0x00005d00 0x000f140f 0x000fe400 +0x0c107211 0x0000000e 0x078010ff 0x000fe400 +0x120e7819 0x00000002 0x000006ff 0x000fe400 +0xff0f7819 0x0000001e 0x00011612 0x000fe400 +0x0c0d7211 0x00000011 0x000f140d 0x000fe400 +0x0e127211 0x00000010 0x078210ff 0x000fc400 +0x0e117210 0x00000010 0x07f1e0ff 0x040fe400 +0x0e137211 0x0000000d 0x008f140f 0x040fe400 +0x0e157210 0x00000012 0x07f3e0ff 0x000fe200 +0x0f147824 0x00000001 0x000e060d 0x040fe400 +0xff0c7224 0x000000ff 0x078e0010 0x000fe400 +0x0f167824 0x00000001 0x008e0613 0x000fe400 +0xff0e7224 0x000000ff 0x078e0011 0x000fc400 +0xff0f7224 0x000000ff 0x078e0014 0x000fe200 +0x00107202 0x00000015 0x00000f00 0x000fe200 +0xff117224 0x000000ff 0x078e0016 0x000fe200 +0x0c007385 0x00000000 0x0010e908 0x0001e200 +0x0c007385 0x00000008 0x0010e90a 0x0003e800 +0x0e007385 0x00000000 0x0010e909 0x0003e200 +0x0e007385 0x00000008 0x0010e90b 0x0003e200 +0x12007385 0x00000000 0x0010e904 0x0003e200 +0x12007385 0x00000008 0x0010e906 0x0003e200 +0x10007385 0x00000000 0x0010e905 0x0003e200 +0x10007385 0x00000008 0x0010e907 0x0003e200 +0x00007948 0xffffffff 0x03800000 0x000fe200 +0x02087a10 0x80000800 0x07ffe0ff 0x001fd000 +0x00047805 0x00000000 0x00005000 0x002fd000 +0x04037824 0x00000001 0x078e0a03 0x000fd000 +0x08007387 0x00000003 0x00100800 0x0001e200 +0xff067224 0x000000ff 0x078e0002 0x000fe200 +0x00047802 0x00000000 0x00000f00 0x000fe200 +0xff077224 0x000000ff 0x078e0000 0x000fe200 +0x00057802 0x00000000 0x00000f00 0x000fe400 +0x00147802 0x00000000 0x00000f00 0x000fe400 +0x00157802 0x00000000 0x00000f00 0x000fd000 +0x00007943 0x00000000 0x03c00000 0x001fea00 +0x0000794d 0x00000000 0x03800000 0x000fea00 +0x00007947 0xfffffff0 0x0383ffff 0x000fc000 +0x00007918 0x00000000 0x00000000 0x000fc000 +0x00007918 0x00000000 0x00000000 0x000fc000 + + +.section .rel.text._Z12wmma_exampleP6__halfS0_Pfiiiff REL +2272 $str R_CUDA_ABS32_LO_32 +2304 $str R_CUDA_ABS32_HI_32 +2352 vprintf R_CUDA_ABS47_34 + +.section .rela.text._Z12wmma_exampleP6__halfS0_Pfiiiff RELA +2320 _Z12wmma_exampleP6__halfS0_Pfiiiff R_CUDA_ABS32_LO_32 2368 +2336 _Z12wmma_exampleP6__halfS0_Pfiiiff R_CUDA_ABS32_HI_32 2368 + +.section .debug_frame +decodeDebugFrame, frameBuf 0xffffffff, total_length 224 +CIE length 40, cie_id -1 +version 3 +augmentation slen 1 +augmentation +code_align_factor slen 1 +data_align_factor slen 1 + Debug Frame Common Information Entry + length: 40 + CIE_id : -1 + version: 3 + augmentation: + code align factor: 4 + data align factor: -4 + return address register 16777215 + initial instructions: 23 bytes, ptr = 0x8080810c, frameBuf = 0xffffffff + DW_CFA_def_cfa register R1, offset 0 + DW_CFA_same_value R255 + DW_CFA_same_value R1 + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + Debug Frame Description Entry + length: 48 + CIE_pointer: 0 + initial_location: 0x0 + address_range: 0x100 + instructions: 24 bytes + DW_CFA_advance_loc4 delta 4 + DW_CFA_advance_loc4 delta 0 + DW_CFA_def_cfa register R1, offset 0 + DW_CFA_advance_loc4 delta 52 + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop +CIE length 40, cie_id -1 +version 3 +augmentation slen 1 +augmentation +code_align_factor slen 1 +data_align_factor slen 1 + Debug Frame Common Information Entry + length: 40 + CIE_id : -1 + version: 3 + augmentation: + code align factor: 4 + data align factor: -4 + return address register 16777215 + initial instructions: 23 bytes, ptr = 0x8080810c, frameBuf = 0xffffffff + DW_CFA_def_cfa register R1, offset 0 + DW_CFA_same_value R255 + DW_CFA_same_value R1 + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + Debug Frame Description Entry + length: 48 + CIE_pointer: 0 + initial_location: 0x0 + address_range: 0x970 + instructions: 24 bytes + DW_CFA_advance_loc4 delta 4 + DW_CFA_advance_loc4 delta 2 + DW_CFA_def_cfa register R1, offset 8 + DW_CFA_advance_loc4 delta 586 + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + +.section .rel.debug_frame REL +72 _Z17convertFp32ToFp16P6__halfPfi R_NV_64 +184 _Z12wmma_exampleP6__halfS0_Pfiiiff R_NV_64 + + code for sm_70 + Function : _Z17convertFp32ToFp16P6__halfPfi + .headerflags @"EF_CUDA_SM70 EF_CUDA_PTX_SM(EF_CUDA_SM70)" + /*0000*/ @!PT SHFL.IDX PT, RZ, RZ, RZ, RZ; /* 0x000000fffffff389 */ + /* 0x000fe200000e00ff */ + /*0010*/ MOV R1, c[0x0][0x28]; /* 0x00000a0000017a02 */ + /* 0x000fd00000000f00 */ + /*0020*/ S2R R4, SR_CTAID.X; /* 0x0000000000047919 */ + /* 0x000e220000002500 */ + /*0030*/ S2R R2, SR_TID.X; /* 0x0000000000027919 */ + /* 0x000e240000002100 */ + /*0040*/ IMAD R4, R4, c[0x0][0x0], R2; /* 0x0000000004047a24 */ + /* 0x001fca00078e0202 */ + /*0050*/ ISETP.GE.AND P0, PT, R4, c[0x0][0x170], PT, !PT; /* 0x00005c0004007a0c */ + /* 0x000fd80003f062f0 */ + /*0060*/ @P0 EXIT; /* 0x000000000000094d */ + /* 0x000fea0003800000 */ + /*0070*/ MOV R2, 0x4; /* 0x0000000400027802 */ + /* 0x000fca0000000f00 */ + /*0080*/ IMAD.WIDE R2, R4, R2, c[0x0][0x168]; /* 0x00005a0004027625 */ + /* 0x000fd400078e0202 */ + /*0090*/ LDG.E.SYS R2, [R2]; /* 0x0000000002027381 */ + /* 0x000e2200001ee900 */ + /*00a0*/ MOV R5, 0x2; /* 0x0000000200057802 */ + /* 0x000fca0000000f00 */ + /*00b0*/ IMAD.WIDE R4, R4, R5, c[0x0][0x160]; /* 0x0000580004047625 */ + /* 0x000fe200078e0205 */ + /*00c0*/ F2F.F16.F32 R6, R2; /* 0x0000000200067304 */ + /* 0x001e320000200800 */ + /*00d0*/ STG.E.U16.SYS [R4], R6; /* 0x0000000604007386 */ + /* 0x0011e2000010e500 */ + /*00e0*/ EXIT; /* 0x000000000000794d */ + /* 0x000fea0003800000 */ + /*00f0*/ BRA 0xf0; /* 0xfffffff000007947 */ + /* 0x000fc0000383ffff */ + ........................................... + + + Function : _Z12wmma_exampleP6__halfS0_Pfiiiff + .headerflags @"EF_CUDA_SM70 EF_CUDA_PTX_SM(EF_CUDA_SM70)" + /*0000*/ @!PT SHFL.IDX PT, RZ, RZ, RZ, RZ; /* 0x000000fffffff389 */ + /* 0x000fe200000e00ff */ + /*0010*/ IMAD.U32 R1, RZ, RZ, c[0x0][0x28]; /* 0x00000a00ff017624 */ + /* 0x000fd000078e00ff */ + /*0020*/ IADD3 R1, R1, -0x8, RZ; /* 0xfffffff801017810 */ + /* 0x000fc80007ffe0ff */ + /*0030*/ IADD3 R2, P0, R1, c[0x0][0x20], RZ; /* 0x0000080001027a10 */ + /* 0x000fca0007f1e0ff */ + /*0040*/ IMAD.X R0, RZ, RZ, c[0x0][0x24], P0; /* 0x00000900ff007624 */ + /* 0x000fd000000e06ff */ + /*0050*/ CS2R.32 R3, SR_CLOCKLO; /* 0x0000000000037805 */ + /* 0x000fd00000005000 */ + /*0060*/ I2F.U32.RP R7, 0x20; /* 0x0000002000077906 */ + /* 0x000e240000209000 */ + /*0070*/ MUFU.RCP R7, R7; /* 0x0000000700077308 */ + /* 0x001e220000001000 */ + /*0080*/ S2R R6, SR_CTAID.X; /* 0x0000000000067919 */ + /* 0x000e620000002500 */ + /*0090*/ S2R R9, SR_TID.X; /* 0x0000000000097919 */ + /* 0x000e620000002100 */ + /*00a0*/ IADD3 R8, R7, 0xffffffe, RZ; /* 0x0ffffffe07087810 */ + /* 0x001fcc0007ffe0ff */ + /*00b0*/ F2I.FTZ.U32.TRUNC.NTZ R5, R8; /* 0x0000000800057305 */ + /* 0x0000a2000021f000 */ + /*00c0*/ IMAD.U32 R4, RZ, RZ, RZ; /* 0x000000ffff047224 */ + /* 0x000fe400078e00ff */ + /*00d0*/ IMAD R6, R6, c[0x0][0x0], R9; /* 0x0000000006067a24 */ + /* 0x002fe400078e0209 */ + /*00e0*/ IMAD.U32 R10, R5, -0x20, RZ; /* 0xffffffe0050a7824 */ + /* 0x004fc800078e00ff */ + /*00f0*/ IMAD.WIDE.U32 R4, R5, R10, R4; /* 0x0000000a05047225 */ + /* 0x000fd000078e0004 */ + /*0100*/ IMAD.WIDE.U32 R4, R5, R6, RZ; /* 0x0000000605047225 */ + /* 0x000fcc00078e00ff */ + /*0110*/ IMAD R4, RZ, RZ, -R5; /* 0x000000ffff047224 */ + /* 0x000fc800078e0a05 */ + /*0120*/ IMAD R6, R4, 0x20, R6; /* 0x0000002004067824 */ + /* 0x000fca00078e0206 */ + /*0130*/ ISETP.GE.U32.AND P0, PT, R6.reuse, 0x20, PT, !PT; /* 0x000000200600780c */ + /* 0x040fe20003f060f0 */ + /*0140*/ S2R R28, SR_CTAID.Y; /* 0x00000000001c7919 */ + /* 0x000e220000002600 */ + /*0150*/ S2R R7, SR_TID.Y; /* 0x0000000000077919 */ + /* 0x000e340000002200 */ + /*0160*/ @P0 IADD3 R6, R6, -0x20, RZ; /* 0xffffffe006060810 */ + /* 0x000fc80007ffe0ff */ + /*0170*/ ISETP.GE.U32.AND P1, PT, R6, 0x20, PT, !PT; /* 0x000000200600780c */ + /* 0x000fe40003f260f0 */ + /*0180*/ @P0 IADD3 R5, R5, 0x1, RZ; /* 0x0000000105050810 */ + /* 0x000fe40007ffe0ff */ + /*0190*/ ISETP.LT.AND P0, PT, RZ, c[0x0][0x180], PT, !PT; /* 0x00006000ff007a0c */ + /* 0x000fd00003f012f0 */ + /*01a0*/ @P1 IADD3 R5, R5, 0x1, RZ; /* 0x0000000105051810 */ + /* 0x000fe20007ffe0ff */ + /*01b0*/ IMAD R28, R28, c[0x0][0x4], R7; /* 0x000001001c1c7a24 */ + /* 0x001fc600078e0207 */ + /*01c0*/ SHF.L.U32 R29, R5, 0x4, RZ; /* 0x00000004051d7819 */ + /* 0x000fe200000006ff */ + /*01d0*/ IMAD.U32 R28, R28, 0x10, RZ; /* 0x000000101c1c7824 */ + /* 0x000fc600078e00ff */ + /*01e0*/ ISETP.LT.AND P0, PT, R29, c[0x0][0x178], P0, !PT; /* 0x00005e001d007a0c */ + /* 0x000fc800007012f0 */ + /*01f0*/ ISETP.LT.AND P0, PT, R28, c[0x0][0x17c], P0, !PT; /* 0x00005f001c007a0c */ + /* 0x000fe200007012f0 */ + /*0200*/ BSSY B0, 0x5b0; /* 0x000003a000007945 */ + /* 0x000fe20003800000 */ + /*0210*/ IMAD.U32 R7, RZ, RZ, RZ; /* 0x000000ffff077224 */ + /* 0x000fe200078e00ff */ + /*0220*/ MOV R11, RZ; /* 0x000000ff000b7202 */ + /* 0x000fe20000000f00 */ + /*0230*/ IMAD.U32 R6, RZ, RZ, RZ; /* 0x000000ffff067224 */ + /* 0x000fe400078e00ff */ + /*0240*/ IMAD.U32 R5, RZ, RZ, RZ; /* 0x000000ffff057224 */ + /* 0x000fe400078e00ff */ + /*0250*/ IMAD.U32 R4, RZ, RZ, RZ; /* 0x000000ffff047224 */ + /* 0x000fe400078e00ff */ + /*0260*/ IMAD.U32 R10, RZ, RZ, RZ; /* 0x000000ffff0a7224 */ + /* 0x000fc400078e00ff */ + /*0270*/ IMAD.U32 R9, RZ, RZ, RZ; /* 0x000000ffff097224 */ + /* 0x000fe400078e00ff */ + /*0280*/ IMAD.U32 R8, RZ, RZ, RZ; /* 0x000000ffff087224 */ + /* 0x000fe200078e00ff */ + /*0290*/ @!P0 BRA 0x5a0; /* 0x0000030000008947 */ + /* 0x000fee0003800000 */ + /*02a0*/ S2R R6, SR_LANEID; /* 0x0000000000067919 */ + /* 0x000e220000000000 */ + /*02b0*/ IMAD.U32 R10, RZ, RZ, 0x2; /* 0x00000002ff0a7424 */ + /* 0x000fc800078e00ff */ + /*02c0*/ IMAD.WIDE R16, R29, R10, c[0x0][0x160]; /* 0x000058001d107625 */ + /* 0x000fe200078e020a */ + /*02d0*/ SHF.R.U32.HI R4, RZ, 0x2, R6; /* 0x00000002ff047819 */ + /* 0x001fc80000011606 */ + /*02e0*/ LOP3.LUT R5, R4, 0x3, RZ, 0xc0, !PT; /* 0x0000000304057812 */ + /* 0x000fe400078ec0ff */ + /*02f0*/ LOP3.LUT R4, R6, 0x3, RZ, 0xc0, !PT; /* 0x0000000306047812 */ + /* 0x000fe400078ec0ff */ + /*0300*/ LOP3.LUT R7, R5, 0x1, RZ, 0xc0, !PT; /* 0x0000000105077812 */ + /* 0x000fe400078ec0ff */ + /*0310*/ SHF.R.U32.HI R6, RZ, 0x4, R6; /* 0x00000004ff067819 */ + /* 0x000fe40000011606 */ + /*0320*/ SHF.R.U32.HI R5, RZ, 0x1, R5; /* 0x00000001ff057819 */ + /* 0x000fe20000011605 */ + /*0330*/ IMAD R7, R7, 0x8, R4; /* 0x0000000807077824 */ + /* 0x000fe200078e0204 */ + /*0340*/ LOP3.LUT R6, R6, 0x1, RZ, 0xc0, !PT; /* 0x0000000106067812 */ + /* 0x000fc400078ec0ff */ + /*0350*/ LEA R4, R5, R4, 0x3; /* 0x0000000405047211 */ + /* 0x000fe200078e18ff */ + /*0360*/ IMAD.WIDE R12, R28, R10, c[0x0][0x168]; /* 0x00005a001c0c7625 */ + /* 0x000fe400078e020a */ + /*0370*/ IMAD R7, R6.reuse, 0x4, R7; /* 0x0000000406077824 */ + /* 0x040fe400078e0207 */ + /*0380*/ IMAD R4, R6, 0x4, R4; /* 0x0000000406047824 */ + /* 0x000fe400078e0204 */ + /*0390*/ IMAD.U32 R7, R7, 0x2, RZ; /* 0x0000000207077824 */ + /* 0x000fe400078e00ff */ + /*03a0*/ IMAD.U32 R5, R4, 0x2, RZ; /* 0x0000000204057824 */ + /* 0x000fe400078e00ff */ + /*03b0*/ IMAD.WIDE.U32 R16, R7, c[0x0][0x178], R16; /* 0x00005e0007107a25 */ + /* 0x000fc400078e0010 */ + /*03c0*/ IMAD.WIDE.U32 R12, R5, c[0x0][0x180], R12; /* 0x00006000050c7a25 */ + /* 0x000fd000078e000c */ + /*03d0*/ LD.E.128.SYS R24, [R16]; /* 0x0000000010187980 */ + /* 0x000064000010ed00 */ + /*03e0*/ LD.E.128.SYS R20, [R12]; /* 0x000000000c147980 */ + /* 0x000462000010ed00 */ + /*03f0*/ LD.E.128.SYS R16, [R16+0x10]; /* 0x0000001010107980 */ + /* 0x001e22000010ed00 */ + /*0400*/ LD.E.128.SYS R12, [R12+0x10]; /* 0x000000100c0c7980 */ + /* 0x004e22000010ed00 */ + /*0410*/ IMAD.U32 R8, RZ, RZ, RZ; /* 0x000000ffff087224 */ + /* 0x000fe200078e00ff */ + /*0420*/ MOV R9, RZ; /* 0x000000ff00097202 */ + /* 0x000fe20000000f00 */ + /*0430*/ IMAD.U32 R10, RZ, RZ, RZ; /* 0x000000ffff0a7224 */ + /* 0x000fe400078e00ff */ + /*0440*/ IMAD.U32 R11, RZ, RZ, RZ; /* 0x000000ffff0b7224 */ + /* 0x000fe200078e00ff */ + /*0450*/ MOV R7, RZ; /* 0x000000ff00077202 */ + /* 0x000fe20000000f00 */ + /*0460*/ IMAD.U32 R4, RZ, RZ, RZ; /* 0x000000ffff047224 */ + /* 0x000fc400078e00ff */ + /*0470*/ IMAD.U32 R5, RZ, RZ, RZ; /* 0x000000ffff057224 */ + /* 0x000fe400078e00ff */ + /*0480*/ IMAD.U32 R6, RZ, RZ, RZ; /* 0x000000ffff067224 */ + /* 0x000fe200078e00ff */ + /*0490*/ WARPSYNC 0xffffffff; /* 0xffffffff00007948 */ + /* 0x000fe20003800000 */ + /*04a0*/ HMMA.884.F32.F32.STEP0 R8, R24.reuse, R20.reuse.T, R8; /* 0x0000001418087236 */ + /* 0x0c22640000005408 */ + /*04b0*/ HMMA.884.F32.F32.STEP1 R10, R24.reuse, R20.reuse.T, R10; /* 0x00000014180a7236 */ + /* 0x0c04a4000000d40a */ + /*04c0*/ HMMA.884.F32.F32.STEP2 R4, R24.reuse, R20.reuse.T, R4; /* 0x0000001418047236 */ + /* 0x0c06e40000015404 */ + /*04d0*/ HMMA.884.F32.F32.STEP3 R6, R24, R20.T, R6; /* 0x0000001418067236 */ + /* 0x000928000001d406 */ + /*04e0*/ HMMA.884.F32.F32.STEP0 R8, R26.reuse, R22.reuse.T, R8; /* 0x000000161a087236 */ + /* 0x0c20240000005408 */ + /*04f0*/ HMMA.884.F32.F32.STEP1 R10, R26.reuse, R22.reuse.T, R10; /* 0x000000161a0a7236 */ + /* 0x0c4264000000d40a */ + /*0500*/ HMMA.884.F32.F32.STEP2 R4, R26.reuse, R22.reuse.T, R4; /* 0x000000161a047236 */ + /* 0x0c84a40000015404 */ + /*0510*/ HMMA.884.F32.F32.STEP3 R6, R26, R22.T, R6; /* 0x000000161a067236 */ + /* 0x0106e8000001d406 */ + /*0520*/ HMMA.884.F32.F32.STEP0 R8, R16.reuse, R12.reuse.T, R8; /* 0x0000000c10087236 */ + /* 0x0c10240000005408 */ + /*0530*/ HMMA.884.F32.F32.STEP1 R10, R16.reuse, R12.reuse.T, R10; /* 0x0000000c100a7236 */ + /* 0x0c2264000000d40a */ + /*0540*/ HMMA.884.F32.F32.STEP2 R4, R16.reuse, R12.reuse.T, R4; /* 0x0000000c10047236 */ + /* 0x0c44a40000015404 */ + /*0550*/ HMMA.884.F32.F32.STEP3 R6, R16, R12.T, R6; /* 0x0000000c10067236 */ + /* 0x0086e8000001d406 */ + /*0560*/ HMMA.884.F32.F32.STEP0 R8, R18.reuse, R14.reuse.T, R8; /* 0x0000000e12087236 */ + /* 0x0c10240000005408 */ + /*0570*/ HMMA.884.F32.F32.STEP1 R10, R18.reuse, R14.reuse.T, R10; /* 0x0000000e120a7236 */ + /* 0x0c2024000000d40a */ + /*0580*/ HMMA.884.F32.F32.STEP2 R4, R18.reuse, R14.reuse.T, R4; /* 0x0000000e12047236 */ + /* 0x0c40240000015404 */ + /*0590*/ HMMA.884.F32.F32.STEP3 R6, R18, R14.T, R6; /* 0x0000000e12067236 */ + /* 0x008034000001d406 */ + /*05a0*/ BSYNC B0; /* 0x0000000000007941 */ + /* 0x001fea0003800000 */ + /*05b0*/ S2R R12, SR_LANEID; /* 0x00000000000c7919 */ + /* 0x000e220000000000 */ + /*05c0*/ IMAD R28, R28, c[0x0][0x178], RZ; /* 0x00005e001c1c7a24 */ + /* 0x000fe200078e02ff */ + /*05d0*/ SHF.R.U32.HI R14, RZ, 0x4, R12; /* 0x00000004ff0e7819 */ + /* 0x001fc4000001160c */ + /*05e0*/ SHF.R.U32.HI R13, RZ, 0x2, R12; /* 0x00000002ff0d7819 */ + /* 0x000fe4000001160c */ + /*05f0*/ LOP3.LUT R12, R12, 0x3, RZ, 0xc0, !PT; /* 0x000000030c0c7812 */ + /* 0x000fe400078ec0ff */ + /*0600*/ LOP3.LUT R14, R14, 0x1, RZ, 0xc0, !PT; /* 0x000000010e0e7812 */ + /* 0x000fe400078ec0ff */ + /*0610*/ LOP3.LUT R13, R13, 0x3, RZ, 0xc0, !PT; /* 0x000000030d0d7812 */ + /* 0x000fc600078ec0ff */ + /*0620*/ IMAD R12, R14, 0x4, R12; /* 0x000000040e0c7824 */ + /* 0x000fe200078e020c */ + /*0630*/ LOP3.LUT R15, R13, 0x1, RZ, 0xc0, !PT; /* 0x000000010d0f7812 */ + /* 0x000fe400078ec0ff */ + /*0640*/ SHF.R.U32.HI R16, RZ, 0x1, R13; /* 0x00000001ff107819 */ + /* 0x000fe4000001160d */ + /*0650*/ LOP3.LUT R13, R12.reuse, 0x5, RZ, 0xc0, !PT; /* 0x000000050c0d7812 */ + /* 0x040fe400078ec0ff */ + /*0660*/ LOP3.LUT R14, R12, 0x2, RZ, 0xc0, !PT; /* 0x000000020c0e7812 */ + /* 0x000fc600078ec0ff */ + /*0670*/ IMAD R12, R15, 0x8, R13; /* 0x000000080f0c7824 */ + /* 0x000fe200078e020d */ + /*0680*/ SHF.R.S32.HI R15, RZ, 0x1f, R29; /* 0x0000001fff0f7819 */ + /* 0x000fe2000001141d */ + /*0690*/ IMAD R14, R16, 0x8, R14; /* 0x00000008100e7824 */ + /* 0x000fe200078e020e */ + /*06a0*/ IADD3 R17, P0, R29, R28, RZ; /* 0x0000001c1d117210 */ + /* 0x000fe20007f1e0ff */ + /*06b0*/ IMAD.U32 R13, RZ, RZ, RZ; /* 0x000000ffff0d7224 */ + /* 0x000fc600078e00ff */ + /*06c0*/ LEA.HI.X.SX32 R15, R28, R15, 0x1, P0; /* 0x0000000f1c0f7211 */ + /* 0x000fe200000f0eff */ + /*06d0*/ IMAD.WIDE.U32 R12, R14, c[0x0][0x178], R12; /* 0x00005e000e0c7a25 */ + /* 0x000fe200078e000c */ + /*06e0*/ LEA R14, P0, R17, c[0x0][0x170], 0x2; /* 0x00005c00110e7a11 */ + /* 0x000fe200078010ff */ + /*06f0*/ IMAD.U32 R18, RZ, RZ, c[0x0][0x178]; /* 0x00005e00ff127624 */ + /* 0x000fc600078e00ff */ + /*0700*/ LEA.HI.X R17, R17, c[0x0][0x174], R15, 0x2, P0; /* 0x00005d0011117a11 */ + /* 0x000fe400000f140f */ + /*0710*/ LEA R16, P0, R12, R14, 0x2; /* 0x0000000e0c107211 */ + /* 0x000fe400078010ff */ + /*0720*/ SHF.L.U32 R14, R18, 0x2, RZ; /* 0x00000002120e7819 */ + /* 0x000fe400000006ff */ + /*0730*/ SHF.R.U32.HI R15, RZ, 0x1e, R18; /* 0x0000001eff0f7819 */ + /* 0x000fe40000011612 */ + /*0740*/ LEA.HI.X R13, R12, R17, R13, 0x2, P0; /* 0x000000110c0d7211 */ + /* 0x000fe400000f140d */ + /*0750*/ LEA R18, P1, R14, R16, 0x2; /* 0x000000100e127211 */ + /* 0x000fc400078210ff */ + /*0760*/ IADD3 R17, P0, R14.reuse, R16, RZ; /* 0x000000100e117210 */ + /* 0x040fe40007f1e0ff */ + /*0770*/ LEA.HI.X R19, R14.reuse, R13, R15, 0x2, P1; /* 0x0000000d0e137211 */ + /* 0x040fe400008f140f */ + /*0780*/ IADD3 R21, P1, R14, R18, RZ; /* 0x000000120e157210 */ + /* 0x000fe20007f3e0ff */ + /*0790*/ IMAD.X R20, R15.reuse, 0x1, R13, P0; /* 0x000000010f147824 */ + /* 0x040fe400000e060d */ + /*07a0*/ IMAD.U32 R12, RZ, RZ, R16; /* 0x000000ffff0c7224 */ + /* 0x000fe400078e0010 */ + /*07b0*/ IMAD.X R22, R15, 0x1, R19, P1; /* 0x000000010f167824 */ + /* 0x000fe400008e0613 */ + /*07c0*/ IMAD.U32 R14, RZ, RZ, R17; /* 0x000000ffff0e7224 */ + /* 0x000fc400078e0011 */ + /*07d0*/ IMAD.U32 R15, RZ, RZ, R20; /* 0x000000ffff0f7224 */ + /* 0x000fe200078e0014 */ + /*07e0*/ MOV R16, R21; /* 0x0000001500107202 */ + /* 0x000fe20000000f00 */ + /*07f0*/ IMAD.U32 R17, RZ, RZ, R22; /* 0x000000ffff117224 */ + /* 0x000fe200078e0016 */ + /*0800*/ ST.E.SYS [R12], R8; /* 0x000000000c007385 */ + /* 0x0001e2000010e908 */ + /*0810*/ ST.E.SYS [R12+0x8], R10; /* 0x000000080c007385 */ + /* 0x0003e8000010e90a */ + /*0820*/ ST.E.SYS [R14], R9; /* 0x000000000e007385 */ + /* 0x0003e2000010e909 */ + /*0830*/ ST.E.SYS [R14+0x8], R11; /* 0x000000080e007385 */ + /* 0x0003e2000010e90b */ + /*0840*/ ST.E.SYS [R18], R4; /* 0x0000000012007385 */ + /* 0x0003e2000010e904 */ + /*0850*/ ST.E.SYS [R18+0x8], R6; /* 0x0000000812007385 */ + /* 0x0003e2000010e906 */ + /*0860*/ ST.E.SYS [R16], R5; /* 0x0000000010007385 */ + /* 0x0003e2000010e905 */ + /*0870*/ ST.E.SYS [R16+0x8], R7; /* 0x0000000810007385 */ + /* 0x0003e2000010e907 */ + /*0880*/ WARPSYNC 0xffffffff; /* 0xffffffff00007948 */ + /* 0x000fe20003800000 */ + /*0890*/ IADD3 R8, R2, -c[0x0][0x20], RZ; /* 0x8000080002087a10 */ + /* 0x001fd00007ffe0ff */ + /*08a0*/ CS2R.32 R4, SR_CLOCKLO; /* 0x0000000000047805 */ + /* 0x002fd00000005000 */ + /*08b0*/ IMAD R3, R4, 0x1, -R3; /* 0x0000000104037824 */ + /* 0x000fd000078e0a03 */ + /*08c0*/ STL [R8], R3; /* 0x0000000308007387 */ + /* 0x0001e20000100800 */ + /*08d0*/ IMAD.U32 R6, RZ, RZ, R2; /* 0x000000ffff067224 */ + /* 0x000fe200078e0002 */ + /*08e0*/ MOV R4, 0x0; /* 0x0000000000047802 */ + /* 0x000fe20000000f00 */ + /*08f0*/ IMAD.U32 R7, RZ, RZ, R0; /* 0x000000ffff077224 */ + /* 0x000fe200078e0000 */ + /*0900*/ MOV R5, 0x0; /* 0x0000000000057802 */ + /* 0x000fe40000000f00 */ + /*0910*/ MOV R20, 0x0; /* 0x0000000000147802 */ + /* 0x000fe40000000f00 */ + /*0920*/ MOV R21, 0x0; /* 0x0000000000157802 */ + /* 0x000fd00000000f00 */ + /*0930*/ CALL.ABS.NOINC 0x0; /* 0x0000000000007943 */ + /* 0x001fea0003c00000 */ + /*0940*/ EXIT; /* 0x000000000000794d */ + /* 0x000fea0003800000 */ + /*0950*/ BRA 0x950; /* 0xfffffff000007947 */ + /* 0x000fc0000383ffff */ + /*0960*/ NOP; /* 0x0000000000007918 */ + /* 0x000fc00000000000 */ + /*0970*/ NOP; /* 0x0000000000007918 */ + /* 0x000fc00000000000 */ + ............................................. + + + +Fatbin ptx code: +================ +arch = sm_70 +code version = [6,0] +producer = cuda +host = linux +compile_size = 64bit +compressed + + + + + + + + +.version 6.0 +.target sm_70 +.address_size 64 + + +.extern .func (.param .b32 func_retval0) vprintf +( +.param .b64 vprintf_param_0, +.param .b64 vprintf_param_1 +) +; +.global .align 16 .b8 $str[9] = {99, 108, 111, 99, 107, 61, 37, 100, 0}; + +.visible .entry _Z12wmma_exampleP6__halfS0_Pfiiiff( +.param .u64 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_0, +.param .u64 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_1, +.param .u64 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_2, +.param .u32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_3, +.param .u32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_4, +.param .u32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_5, +.param .f32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_6, +.param .f32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_7 +) +{ +.local .align 8 .b8 __local_depot0[8]; +.reg .b64 %SP; +.reg .b64 %SPL; +.reg .pred %p<6>; +.reg .f32 %f<34>; +.reg .b32 %r<38>; +.reg .b64 %rd<18>; + + +mov.u64 %rd17, __local_depot0; +cvta.local.u64 %SP, %rd17; +ld.param.u64 %rd1, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_0]; +ld.param.u64 %rd2, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_1]; +ld.param.u64 %rd3, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_2]; +ld.param.u32 %r4, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_3]; +ld.param.u32 %r7, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_4]; +ld.param.u32 %r5, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_5]; + + mov.u32 %r6, %clock; + + mov.u32 %r8, %ntid.x; +mov.u32 %r9, %ctaid.x; +mov.u32 %r10, %tid.x; +mad.lo.s32 %r11, %r8, %r9, %r10; +mov.u32 %r12, WARP_SZ; +div.u32 %r13, %r11, %r12; +mov.u32 %r14, %ntid.y; +mov.u32 %r15, %ctaid.y; +mov.u32 %r16, %tid.y; +mad.lo.s32 %r17, %r14, %r15, %r16; +shl.b32 %r2, %r13, 4; +shl.b32 %r3, %r17, 4; +setp.lt.s32 %p1, %r2, %r4; +setp.gt.s32 %p2, %r5, 0; +and.pred %p3, %p1, %p2; +setp.lt.s32 %p4, %r3, %r7; +and.pred %p5, %p3, %p4; +mov.f32 %f26, 0f00000000; +mov.f32 %f27, %f26; +mov.f32 %f28, %f26; +mov.f32 %f29, %f26; +mov.f32 %f30, %f26; +mov.f32 %f31, %f26; +mov.f32 %f32, %f26; +mov.f32 %f33, %f26; +@!%p5 bra BB0_2; +bra.uni BB0_1; + +BB0_1: +mul.wide.s32 %rd4, %r2, 2; +add.s64 %rd5, %rd1, %rd4; +wmma.load.a.sync.row.m16n16k16.f16 {%r18, %r19, %r20, %r21, %r22, %r23, %r24, %r25}, [%rd5], %r4; +mul.wide.s32 %rd6, %r3, 2; +add.s64 %rd7, %rd2, %rd6; +wmma.load.b.sync.col.m16n16k16.f16 {%r26, %r27, %r28, %r29, %r30, %r31, %r32, %r33}, [%rd7], %r5; +mov.f32 %f25, 0f00000000; +wmma.mma.sync.row.col.m16n16k16.f32.f32 {%f33, %f32, %f31, %f30, %f29, %f28, %f27, %f26}, {%r18, %r19, %r20, %r21, %r22, %r23, %r24, %r25}, {%r26, %r27, %r28, %r29, %r30, %r31, %r32, %r33}, {%f25, %f25, %f25, %f25, %f25, %f25, %f25, %f25}; + +BB0_2: +add.u64 %rd8, %SP, 0; +cvta.to.local.u64 %rd9, %rd8; +mul.lo.s32 %r35, %r3, %r4; +cvt.s64.s32 %rd10, %r35; +cvt.s64.s32 %rd11, %r2; +add.s64 %rd12, %rd10, %rd11; +shl.b64 %rd13, %rd12, 2; +add.s64 %rd14, %rd3, %rd13; +wmma.store.d.sync.col.m16n16k16.f32 [%rd14], {%f33, %f32, %f31, %f30, %f29, %f28, %f27, %f26}, %r4; + + mov.u32 %r34, %clock; + + sub.s32 %r36, %r34, %r6; +st.local.u32 [%rd9], %r36; +mov.u64 %rd15, $str; +cvta.global.u64 %rd16, %rd15; + + { +.reg .b32 temp_param_reg; + + .param .b64 param0; +st.param.b64 [param0+0], %rd16; +.param .b64 param1; +st.param.b64 [param1+0], %rd8; +.param .b32 retval0; +call.uni (retval0), +vprintf, +( +param0, +param1 +); +ld.param.b32 %r37, [retval0+0]; + + + } + ret; +} + + +.visible .entry _Z17convertFp32ToFp16P6__halfPfi( +.param .u64 _Z17convertFp32ToFp16P6__halfPfi_param_0, +.param .u64 _Z17convertFp32ToFp16P6__halfPfi_param_1, +.param .u32 _Z17convertFp32ToFp16P6__halfPfi_param_2 +) +{ +.reg .pred %p<2>; +.reg .b16 %rs<2>; +.reg .f32 %f<2>; +.reg .b32 %r<6>; +.reg .b64 %rd<9>; + + +ld.param.u64 %rd1, [_Z17convertFp32ToFp16P6__halfPfi_param_0]; +ld.param.u64 %rd2, [_Z17convertFp32ToFp16P6__halfPfi_param_1]; +ld.param.u32 %r2, [_Z17convertFp32ToFp16P6__halfPfi_param_2]; +mov.u32 %r3, %ntid.x; +mov.u32 %r4, %ctaid.x; +mov.u32 %r5, %tid.x; +mad.lo.s32 %r1, %r4, %r3, %r5; +setp.ge.s32 %p1, %r1, %r2; +@%p1 bra BB1_2; + +cvta.to.global.u64 %rd3, %rd2; +mul.wide.s32 %rd4, %r1, 4; +add.s64 %rd5, %rd3, %rd4; +ld.global.f32 %f1, [%rd5]; + + { cvt.rn.f16.f32 %rs1, %f1;} + + + cvta.to.global.u64 %rd6, %rd1; +mul.wide.s32 %rd7, %r1, 2; +add.s64 %rd8, %rd6, %rd7; +st.global.u16 [%rd8], %rs1; + +BB1_2: +ret; +} + + diff --git a/cuda-kernels/_cuobjdump_complete_output_rndQyq b/cuda-kernels/_cuobjdump_complete_output_rndQyq new file mode 100644 index 0000000..36999c0 --- /dev/null +++ b/cuda-kernels/_cuobjdump_complete_output_rndQyq @@ -0,0 +1,1055 @@ + +Fatbin elf code: +================ +arch = sm_70 +code version = [1,7] +producer = +host = linux +compile_size = 64bit + +64bit elf: type=2, abi=7, sm=70, toolkit=90, flags = 0x460546 +Sections: +Index Offset Size ES Align Type Flags Link Info Name + 1 40 32 0 1 STRTAB 0 0 0 .shstrtab + 2 72 32 0 1 STRTAB 0 0 0 .strtab + 3 a8 18 18 8 SYMTAB 0 2 0 .symtab + +.section .strtab + +.section .shstrtab + +.section .symtab + index value size info other shndx name + 0 0 0 0 0 0 (null) + + code for sm_70 + +Fatbin elf code: +================ +arch = sm_70 +code version = [1,7] +producer = cuda +host = linux +compile_size = 64bit + +64bit elf: type=2, abi=7, sm=70, toolkit=90, flags = 0x460546 +Sections: +Index Offset Size ES Align Type Flags Link Info Name + 1 40 21b 0 1 STRTAB 0 0 0 .shstrtab + 2 25b 273 0 1 STRTAB 0 0 0 .strtab + 3 4d0 108 18 8 SYMTAB 0 2 7 .symtab + 4 5d8 e0 0 1 PROGBITS 0 0 0 .debug_frame + 5 6b8 48 0 4 CUDA_INFO 0 3 0 .nv.info + 6 700 50 0 4 CUDA_INFO 0 3 d .nv.info._Z17convertFp32ToFp16P6__halfPfi + 7 750 ac 0 4 CUDA_INFO 0 3 e .nv.info._Z12wmma_exampleP6__halfS0_Pfiiiff + 8 800 30 10 8 REL 0 3 e .rel.text._Z12wmma_exampleP6__halfS0_Pfiiiff + 9 830 30 18 8 RELA 0 3 e .rela.text._Z12wmma_exampleP6__halfS0_Pfiiiff + a 860 20 10 8 REL 0 3 4 .rel.debug_frame + b 880 174 0 4 PROGBITS 2 0 d .nv.constant0._Z17convertFp32ToFp16P6__halfPfi + c 9f4 18c 0 4 PROGBITS 2 0 e .nv.constant0._Z12wmma_exampleP6__halfS0_Pfiiiff + d b80 100 0 80 PROGBITS 6 3 9000008 .text._Z17convertFp32ToFp16P6__halfPfi + e c80 980 0 80 PROGBITS 6 3 20000009 .text._Z12wmma_exampleP6__halfS0_Pfiiiff + f 1600 9 0 10 PROGBITS 3 0 0 .nv.global.init + +.section .strtab + +.section .shstrtab + +.section .symtab + index value size info other shndx name + 0 0 0 0 0 0 (null) + 1 0 0 3 0 d .text._Z17convertFp32ToFp16P6__halfPfi + 2 0 0 3 0 f .nv.global.init + 3 0 9 1 0 f $str + 4 0 0 3 0 b .nv.constant0._Z17convertFp32ToFp16P6__halfPfi + 5 0 0 3 0 e .text._Z12wmma_exampleP6__halfS0_Pfiiiff + 6 0 0 3 0 c .nv.constant0._Z12wmma_exampleP6__halfS0_Pfiiiff + 7 0 0 3 0 4 .debug_frame + 8 0 256 12 10 d _Z17convertFp32ToFp16P6__halfPfi + 9 0 2432 12 10 e _Z12wmma_exampleP6__halfS0_Pfiiiff + 10 0 0 12 0 0 vprintf + + +.nv.constant0._Z17convertFp32ToFp16P6__halfPfi +0x00000000 0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 + + + +.nv.constant0._Z12wmma_exampleP6__halfS0_Pfiiiff +0x00000000 0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 0x00000000 0x00000000 +0x00000000 0x00000000 + + +.nv.global.init +0x636f6c63 0x64253d6b 0 + + +.nv.info + <0x1> + Attribute: EIATTR_MAX_STACK_SIZE + Format: EIFMT_SVAL + Value: 0x9 0x0 + <0x2> + Attribute: EIATTR_MIN_STACK_SIZE + Format: EIFMT_SVAL + Value: function: _Z12wmma_exampleP6__halfS0_Pfiiiff(0x9) min stack size: 0x8 + <0x3> + Attribute: EIATTR_FRAME_SIZE + Format: EIFMT_SVAL + Value: function: _Z12wmma_exampleP6__halfS0_Pfiiiff(0x9) frame size: 0x8 + <0x4> + Attribute: EIATTR_MAX_STACK_SIZE + Format: EIFMT_SVAL + Value: 0x8 0x0 + <0x5> + Attribute: EIATTR_MIN_STACK_SIZE + Format: EIFMT_SVAL + Value: function: _Z17convertFp32ToFp16P6__halfPfi(0x8) min stack size: 0x0 + <0x6> + Attribute: EIATTR_FRAME_SIZE + Format: EIFMT_SVAL + Value: function: _Z17convertFp32ToFp16P6__halfPfi(0x8) frame size: 0x0 + + +.nv.info._Z17convertFp32ToFp16P6__halfPfi + <0x1> + Attribute: EIATTR_PARAM_CBANK + Format: EIFMT_SVAL + Value: 0x4 0x140160 + <0x2> + Attribute: EIATTR_CBANK_PARAM_SIZE + Format: EIFMT_HVAL + Value: 0x14 + <0x3> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x2 Offset : 0x10 Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x4> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x1 Offset : 0x8 Size : 0x8 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x5> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x0 Offset : 0x0 Size : 0x8 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x6> + Attribute: EIATTR_MAXREG_COUNT + Format: EIFMT_HVAL + Value: 0xff + <0x7> + Attribute: EIATTR_EXIT_INSTR_OFFSETS + Format: EIFMT_SVAL + Value: 0x60 0xe0 + + +.nv.info._Z12wmma_exampleP6__halfS0_Pfiiiff + <0x1> + Attribute: EIATTR_PARAM_CBANK + Format: EIFMT_SVAL + Value: 0x6 0x2c0160 + <0x2> + Attribute: EIATTR_CBANK_PARAM_SIZE + Format: EIFMT_HVAL + Value: 0x2c + <0x3> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x7 Offset : 0x28 Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x4> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x6 Offset : 0x24 Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x5> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x5 Offset : 0x20 Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x6> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x4 Offset : 0x1c Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x7> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x3 Offset : 0x18 Size : 0x4 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x8> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x2 Offset : 0x10 Size : 0x8 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x9> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x1 Offset : 0x8 Size : 0x8 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x10> + Attribute: EIATTR_KPARAM_INFO + Format: EIFMT_SVAL + Value: Index : 0x0 Ordinal : 0x0 Offset : 0x0 Size : 0x8 + Pointee's logAlignment : 0x0 Space : 0x0 cbank : 0x1f Parameter Space : CBANK + <0x11> + Attribute: EIATTR_MAXREG_COUNT + Format: EIFMT_HVAL + Value: 0xff + <0x12> + Attribute: EIATTR_EXIT_INSTR_OFFSETS + Format: EIFMT_SVAL + Value: 0x940 + <0x13> + Attribute: EIATTR_EXTERNS + Format: EIFMT_SVAL + Value: externs: vprintf(0xa) + <0x14> + Attribute: EIATTR_CRS_STACK_SIZE + Format: EIFMT_SVAL + Value: 0x0 + + +.text._Z17convertFp32ToFp16P6__halfPfi +bar = 0 reg = 9 lmem=0 smem=0 +0xfffff389 0x000000ff 0x000e00ff 0x000fe200 +0x00017a02 0x00000a00 0x00000f00 0x000fd000 +0x00047919 0x00000000 0x00002500 0x000e2200 +0x00027919 0x00000000 0x00002100 0x000e2400 +0x04047a24 0x00000000 0x078e0202 0x001fca00 +0x04007a0c 0x00005c00 0x03f062f0 0x000fd800 +0x0000094d 0x00000000 0x03800000 0x000fea00 +0x00027802 0x00000004 0x00000f00 0x000fca00 +0x04027625 0x00005a00 0x078e0202 0x000fd400 +0x02027381 0x00000000 0x001ee900 0x000e2200 +0x00057802 0x00000002 0x00000f00 0x000fca00 +0x04047625 0x00005800 0x078e0205 0x000fe200 +0x00067304 0x00000002 0x00200800 0x001e3200 +0x04007386 0x00000006 0x0010e500 0x0011e200 +0x0000794d 0x00000000 0x03800000 0x000fea00 +0x00007947 0xfffffff0 0x0383ffff 0x000fc000 + + + +.text._Z12wmma_exampleP6__halfS0_Pfiiiff +bar = 0 reg = 32 lmem=0 smem=0 +0xfffff389 0x000000ff 0x000e00ff 0x000fe200 +0xff017624 0x00000a00 0x078e00ff 0x000fd000 +0x01017810 0xfffffff8 0x07ffe0ff 0x000fc800 +0x01027a10 0x00000800 0x07f1e0ff 0x000fca00 +0xff007624 0x00000900 0x000e06ff 0x000fd000 +0x00037805 0x00000000 0x00005000 0x000fd000 +0x00077906 0x00000020 0x00209000 0x000e2400 +0x00077308 0x00000007 0x00001000 0x001e2200 +0x00067919 0x00000000 0x00002500 0x000e6200 +0x00097919 0x00000000 0x00002100 0x000e6200 +0x07087810 0x0ffffffe 0x07ffe0ff 0x001fcc00 +0x00057305 0x00000008 0x0021f000 0x0000a200 +0xff047224 0x000000ff 0x078e00ff 0x000fe400 +0x06067a24 0x00000000 0x078e0209 0x002fe400 +0x050a7824 0xffffffe0 0x078e00ff 0x004fc800 +0x05047225 0x0000000a 0x078e0004 0x000fd000 +0x05047225 0x00000006 0x078e00ff 0x000fcc00 +0xff047224 0x000000ff 0x078e0a05 0x000fc800 +0x04067824 0x00000020 0x078e0206 0x000fca00 +0x0600780c 0x00000020 0x03f060f0 0x040fe200 +0x001c7919 0x00000000 0x00002600 0x000e2200 +0x00077919 0x00000000 0x00002200 0x000e3400 +0x06060810 0xffffffe0 0x07ffe0ff 0x000fc800 +0x0600780c 0x00000020 0x03f260f0 0x000fe400 +0x05050810 0x00000001 0x07ffe0ff 0x000fe400 +0xff007a0c 0x00006000 0x03f012f0 0x000fd000 +0x05051810 0x00000001 0x07ffe0ff 0x000fe200 +0x1c1c7a24 0x00000100 0x078e0207 0x001fc600 +0x051d7819 0x00000004 0x000006ff 0x000fe200 +0x1c1c7824 0x00000010 0x078e00ff 0x000fc600 +0x1d007a0c 0x00005e00 0x007012f0 0x000fc800 +0x1c007a0c 0x00005f00 0x007012f0 0x000fe200 +0x00007945 0x000003a0 0x03800000 0x000fe200 +0xff077224 0x000000ff 0x078e00ff 0x000fe200 +0x000b7202 0x000000ff 0x00000f00 0x000fe200 +0xff067224 0x000000ff 0x078e00ff 0x000fe400 +0xff057224 0x000000ff 0x078e00ff 0x000fe400 +0xff047224 0x000000ff 0x078e00ff 0x000fe400 +0xff0a7224 0x000000ff 0x078e00ff 0x000fc400 +0xff097224 0x000000ff 0x078e00ff 0x000fe400 +0xff087224 0x000000ff 0x078e00ff 0x000fe200 +0x00008947 0x00000300 0x03800000 0x000fee00 +0x00067919 0x00000000 0x00000000 0x000e2200 +0xff0a7424 0x00000002 0x078e00ff 0x000fc800 +0x1d107625 0x00005800 0x078e020a 0x000fe200 +0xff047819 0x00000002 0x00011606 0x001fc800 +0x04057812 0x00000003 0x078ec0ff 0x000fe400 +0x06047812 0x00000003 0x078ec0ff 0x000fe400 +0x05077812 0x00000001 0x078ec0ff 0x000fe400 +0xff067819 0x00000004 0x00011606 0x000fe400 +0xff057819 0x00000001 0x00011605 0x000fe200 +0x07077824 0x00000008 0x078e0204 0x000fe200 +0x06067812 0x00000001 0x078ec0ff 0x000fc400 +0x05047211 0x00000004 0x078e18ff 0x000fe200 +0x1c0c7625 0x00005a00 0x078e020a 0x000fe400 +0x06077824 0x00000004 0x078e0207 0x040fe400 +0x06047824 0x00000004 0x078e0204 0x000fe400 +0x07077824 0x00000002 0x078e00ff 0x000fe400 +0x04057824 0x00000002 0x078e00ff 0x000fe400 +0x07107a25 0x00005e00 0x078e0010 0x000fc400 +0x050c7a25 0x00006000 0x078e000c 0x000fd000 +0x10187980 0x00000000 0x0010ed00 0x00006400 +0x0c147980 0x00000000 0x0010ed00 0x00046200 +0x10107980 0x00000010 0x0010ed00 0x001e2200 +0x0c0c7980 0x00000010 0x0010ed00 0x004e2200 +0xff087224 0x000000ff 0x078e00ff 0x000fe200 +0x00097202 0x000000ff 0x00000f00 0x000fe200 +0xff0a7224 0x000000ff 0x078e00ff 0x000fe400 +0xff0b7224 0x000000ff 0x078e00ff 0x000fe200 +0x00077202 0x000000ff 0x00000f00 0x000fe200 +0xff047224 0x000000ff 0x078e00ff 0x000fc400 +0xff057224 0x000000ff 0x078e00ff 0x000fe400 +0xff067224 0x000000ff 0x078e00ff 0x000fe200 +0x00007948 0xffffffff 0x03800000 0x000fe200 +0x18087236 0x00000014 0x00005408 0x0c226400 +0x180a7236 0x00000014 0x0000d40a 0x0c04a400 +0x18047236 0x00000014 0x00015404 0x0c06e400 +0x18067236 0x00000014 0x0001d406 0x00092800 +0x1a087236 0x00000016 0x00005408 0x0c202400 +0x1a0a7236 0x00000016 0x0000d40a 0x0c426400 +0x1a047236 0x00000016 0x00015404 0x0c84a400 +0x1a067236 0x00000016 0x0001d406 0x0106e800 +0x10087236 0x0000000c 0x00005408 0x0c102400 +0x100a7236 0x0000000c 0x0000d40a 0x0c226400 +0x10047236 0x0000000c 0x00015404 0x0c44a400 +0x10067236 0x0000000c 0x0001d406 0x0086e800 +0x12087236 0x0000000e 0x00005408 0x0c102400 +0x120a7236 0x0000000e 0x0000d40a 0x0c202400 +0x12047236 0x0000000e 0x00015404 0x0c402400 +0x12067236 0x0000000e 0x0001d406 0x00803400 +0x00007941 0x00000000 0x03800000 0x001fea00 +0x000c7919 0x00000000 0x00000000 0x000e2200 +0x1c1c7a24 0x00005e00 0x078e02ff 0x000fe200 +0xff0e7819 0x00000004 0x0001160c 0x001fc400 +0xff0d7819 0x00000002 0x0001160c 0x000fe400 +0x0c0c7812 0x00000003 0x078ec0ff 0x000fe400 +0x0e0e7812 0x00000001 0x078ec0ff 0x000fe400 +0x0d0d7812 0x00000003 0x078ec0ff 0x000fc600 +0x0e0c7824 0x00000004 0x078e020c 0x000fe200 +0x0d0f7812 0x00000001 0x078ec0ff 0x000fe400 +0xff107819 0x00000001 0x0001160d 0x000fe400 +0x0c0d7812 0x00000005 0x078ec0ff 0x040fe400 +0x0c0e7812 0x00000002 0x078ec0ff 0x000fc600 +0x0f0c7824 0x00000008 0x078e020d 0x000fe200 +0xff0f7819 0x0000001f 0x0001141d 0x000fe200 +0x100e7824 0x00000008 0x078e020e 0x000fe200 +0x1d117210 0x0000001c 0x07f1e0ff 0x000fe200 +0xff0d7224 0x000000ff 0x078e00ff 0x000fc600 +0x1c0f7211 0x0000000f 0x000f0eff 0x000fe200 +0x0e0c7a25 0x00005e00 0x078e000c 0x000fe200 +0x110e7a11 0x00005c00 0x078010ff 0x000fe200 +0xff127624 0x00005e00 0x078e00ff 0x000fc600 +0x11117a11 0x00005d00 0x000f140f 0x000fe400 +0x0c107211 0x0000000e 0x078010ff 0x000fe400 +0x120e7819 0x00000002 0x000006ff 0x000fe400 +0xff0f7819 0x0000001e 0x00011612 0x000fe400 +0x0c0d7211 0x00000011 0x000f140d 0x000fe400 +0x0e127211 0x00000010 0x078210ff 0x000fc400 +0x0e117210 0x00000010 0x07f1e0ff 0x040fe400 +0x0e137211 0x0000000d 0x008f140f 0x040fe400 +0x0e157210 0x00000012 0x07f3e0ff 0x000fe200 +0x0f147824 0x00000001 0x000e060d 0x040fe400 +0xff0c7224 0x000000ff 0x078e0010 0x000fe400 +0x0f167824 0x00000001 0x008e0613 0x000fe400 +0xff0e7224 0x000000ff 0x078e0011 0x000fc400 +0xff0f7224 0x000000ff 0x078e0014 0x000fe200 +0x00107202 0x00000015 0x00000f00 0x000fe200 +0xff117224 0x000000ff 0x078e0016 0x000fe200 +0x0c007385 0x00000000 0x0010e908 0x0001e200 +0x0c007385 0x00000008 0x0010e90a 0x0003e800 +0x0e007385 0x00000000 0x0010e909 0x0003e200 +0x0e007385 0x00000008 0x0010e90b 0x0003e200 +0x12007385 0x00000000 0x0010e904 0x0003e200 +0x12007385 0x00000008 0x0010e906 0x0003e200 +0x10007385 0x00000000 0x0010e905 0x0003e200 +0x10007385 0x00000008 0x0010e907 0x0003e200 +0x00007948 0xffffffff 0x03800000 0x000fe200 +0x02087a10 0x80000800 0x07ffe0ff 0x001fd000 +0x00047805 0x00000000 0x00005000 0x002fd000 +0x04037824 0x00000001 0x078e0a03 0x000fd000 +0x08007387 0x00000003 0x00100800 0x0001e200 +0xff067224 0x000000ff 0x078e0002 0x000fe200 +0x00047802 0x00000000 0x00000f00 0x000fe200 +0xff077224 0x000000ff 0x078e0000 0x000fe200 +0x00057802 0x00000000 0x00000f00 0x000fe400 +0x00147802 0x00000000 0x00000f00 0x000fe400 +0x00157802 0x00000000 0x00000f00 0x000fd000 +0x00007943 0x00000000 0x03c00000 0x001fea00 +0x0000794d 0x00000000 0x03800000 0x000fea00 +0x00007947 0xfffffff0 0x0383ffff 0x000fc000 +0x00007918 0x00000000 0x00000000 0x000fc000 +0x00007918 0x00000000 0x00000000 0x000fc000 + + +.section .rel.text._Z12wmma_exampleP6__halfS0_Pfiiiff REL +2272 $str R_CUDA_ABS32_LO_32 +2304 $str R_CUDA_ABS32_HI_32 +2352 vprintf R_CUDA_ABS47_34 + +.section .rela.text._Z12wmma_exampleP6__halfS0_Pfiiiff RELA +2320 _Z12wmma_exampleP6__halfS0_Pfiiiff R_CUDA_ABS32_LO_32 2368 +2336 _Z12wmma_exampleP6__halfS0_Pfiiiff R_CUDA_ABS32_HI_32 2368 + +.section .debug_frame +decodeDebugFrame, frameBuf 0xffffffff, total_length 224 +CIE length 40, cie_id -1 +version 3 +augmentation slen 1 +augmentation +code_align_factor slen 1 +data_align_factor slen 1 + Debug Frame Common Information Entry + length: 40 + CIE_id : -1 + version: 3 + augmentation: + code align factor: 4 + data align factor: -4 + return address register 16777215 + initial instructions: 23 bytes, ptr = 0x8080810c, frameBuf = 0xffffffff + DW_CFA_def_cfa register R1, offset 0 + DW_CFA_same_value R255 + DW_CFA_same_value R1 + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + Debug Frame Description Entry + length: 48 + CIE_pointer: 0 + initial_location: 0x0 + address_range: 0x100 + instructions: 24 bytes + DW_CFA_advance_loc4 delta 4 + DW_CFA_advance_loc4 delta 0 + DW_CFA_def_cfa register R1, offset 0 + DW_CFA_advance_loc4 delta 52 + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop +CIE length 40, cie_id -1 +version 3 +augmentation slen 1 +augmentation +code_align_factor slen 1 +data_align_factor slen 1 + Debug Frame Common Information Entry + length: 40 + CIE_id : -1 + version: 3 + augmentation: + code align factor: 4 + data align factor: -4 + return address register 16777215 + initial instructions: 23 bytes, ptr = 0x8080810c, frameBuf = 0xffffffff + DW_CFA_def_cfa register R1, offset 0 + DW_CFA_same_value R255 + DW_CFA_same_value R1 + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + Debug Frame Description Entry + length: 48 + CIE_pointer: 0 + initial_location: 0x0 + address_range: 0x970 + instructions: 24 bytes + DW_CFA_advance_loc4 delta 4 + DW_CFA_advance_loc4 delta 2 + DW_CFA_def_cfa register R1, offset 8 + DW_CFA_advance_loc4 delta 586 + DW_CFA_nop + DW_CFA_nop + DW_CFA_nop + +.section .rel.debug_frame REL +72 _Z17convertFp32ToFp16P6__halfPfi R_NV_64 +184 _Z12wmma_exampleP6__halfS0_Pfiiiff R_NV_64 + + code for sm_70 + Function : _Z17convertFp32ToFp16P6__halfPfi + .headerflags @"EF_CUDA_SM70 EF_CUDA_PTX_SM(EF_CUDA_SM70)" + /*0000*/ @!PT SHFL.IDX PT, RZ, RZ, RZ, RZ; /* 0x000000fffffff389 */ + /* 0x000fe200000e00ff */ + /*0010*/ MOV R1, c[0x0][0x28]; /* 0x00000a0000017a02 */ + /* 0x000fd00000000f00 */ + /*0020*/ S2R R4, SR_CTAID.X; /* 0x0000000000047919 */ + /* 0x000e220000002500 */ + /*0030*/ S2R R2, SR_TID.X; /* 0x0000000000027919 */ + /* 0x000e240000002100 */ + /*0040*/ IMAD R4, R4, c[0x0][0x0], R2; /* 0x0000000004047a24 */ + /* 0x001fca00078e0202 */ + /*0050*/ ISETP.GE.AND P0, PT, R4, c[0x0][0x170], PT, !PT; /* 0x00005c0004007a0c */ + /* 0x000fd80003f062f0 */ + /*0060*/ @P0 EXIT; /* 0x000000000000094d */ + /* 0x000fea0003800000 */ + /*0070*/ MOV R2, 0x4; /* 0x0000000400027802 */ + /* 0x000fca0000000f00 */ + /*0080*/ IMAD.WIDE R2, R4, R2, c[0x0][0x168]; /* 0x00005a0004027625 */ + /* 0x000fd400078e0202 */ + /*0090*/ LDG.E.SYS R2, [R2]; /* 0x0000000002027381 */ + /* 0x000e2200001ee900 */ + /*00a0*/ MOV R5, 0x2; /* 0x0000000200057802 */ + /* 0x000fca0000000f00 */ + /*00b0*/ IMAD.WIDE R4, R4, R5, c[0x0][0x160]; /* 0x0000580004047625 */ + /* 0x000fe200078e0205 */ + /*00c0*/ F2F.F16.F32 R6, R2; /* 0x0000000200067304 */ + /* 0x001e320000200800 */ + /*00d0*/ STG.E.U16.SYS [R4], R6; /* 0x0000000604007386 */ + /* 0x0011e2000010e500 */ + /*00e0*/ EXIT; /* 0x000000000000794d */ + /* 0x000fea0003800000 */ + /*00f0*/ BRA 0xf0; /* 0xfffffff000007947 */ + /* 0x000fc0000383ffff */ + ........................................... + + + Function : _Z12wmma_exampleP6__halfS0_Pfiiiff + .headerflags @"EF_CUDA_SM70 EF_CUDA_PTX_SM(EF_CUDA_SM70)" + /*0000*/ @!PT SHFL.IDX PT, RZ, RZ, RZ, RZ; /* 0x000000fffffff389 */ + /* 0x000fe200000e00ff */ + /*0010*/ IMAD.U32 R1, RZ, RZ, c[0x0][0x28]; /* 0x00000a00ff017624 */ + /* 0x000fd000078e00ff */ + /*0020*/ IADD3 R1, R1, -0x8, RZ; /* 0xfffffff801017810 */ + /* 0x000fc80007ffe0ff */ + /*0030*/ IADD3 R2, P0, R1, c[0x0][0x20], RZ; /* 0x0000080001027a10 */ + /* 0x000fca0007f1e0ff */ + /*0040*/ IMAD.X R0, RZ, RZ, c[0x0][0x24], P0; /* 0x00000900ff007624 */ + /* 0x000fd000000e06ff */ + /*0050*/ CS2R.32 R3, SR_CLOCKLO; /* 0x0000000000037805 */ + /* 0x000fd00000005000 */ + /*0060*/ I2F.U32.RP R7, 0x20; /* 0x0000002000077906 */ + /* 0x000e240000209000 */ + /*0070*/ MUFU.RCP R7, R7; /* 0x0000000700077308 */ + /* 0x001e220000001000 */ + /*0080*/ S2R R6, SR_CTAID.X; /* 0x0000000000067919 */ + /* 0x000e620000002500 */ + /*0090*/ S2R R9, SR_TID.X; /* 0x0000000000097919 */ + /* 0x000e620000002100 */ + /*00a0*/ IADD3 R8, R7, 0xffffffe, RZ; /* 0x0ffffffe07087810 */ + /* 0x001fcc0007ffe0ff */ + /*00b0*/ F2I.FTZ.U32.TRUNC.NTZ R5, R8; /* 0x0000000800057305 */ + /* 0x0000a2000021f000 */ + /*00c0*/ IMAD.U32 R4, RZ, RZ, RZ; /* 0x000000ffff047224 */ + /* 0x000fe400078e00ff */ + /*00d0*/ IMAD R6, R6, c[0x0][0x0], R9; /* 0x0000000006067a24 */ + /* 0x002fe400078e0209 */ + /*00e0*/ IMAD.U32 R10, R5, -0x20, RZ; /* 0xffffffe0050a7824 */ + /* 0x004fc800078e00ff */ + /*00f0*/ IMAD.WIDE.U32 R4, R5, R10, R4; /* 0x0000000a05047225 */ + /* 0x000fd000078e0004 */ + /*0100*/ IMAD.WIDE.U32 R4, R5, R6, RZ; /* 0x0000000605047225 */ + /* 0x000fcc00078e00ff */ + /*0110*/ IMAD R4, RZ, RZ, -R5; /* 0x000000ffff047224 */ + /* 0x000fc800078e0a05 */ + /*0120*/ IMAD R6, R4, 0x20, R6; /* 0x0000002004067824 */ + /* 0x000fca00078e0206 */ + /*0130*/ ISETP.GE.U32.AND P0, PT, R6.reuse, 0x20, PT, !PT; /* 0x000000200600780c */ + /* 0x040fe20003f060f0 */ + /*0140*/ S2R R28, SR_CTAID.Y; /* 0x00000000001c7919 */ + /* 0x000e220000002600 */ + /*0150*/ S2R R7, SR_TID.Y; /* 0x0000000000077919 */ + /* 0x000e340000002200 */ + /*0160*/ @P0 IADD3 R6, R6, -0x20, RZ; /* 0xffffffe006060810 */ + /* 0x000fc80007ffe0ff */ + /*0170*/ ISETP.GE.U32.AND P1, PT, R6, 0x20, PT, !PT; /* 0x000000200600780c */ + /* 0x000fe40003f260f0 */ + /*0180*/ @P0 IADD3 R5, R5, 0x1, RZ; /* 0x0000000105050810 */ + /* 0x000fe40007ffe0ff */ + /*0190*/ ISETP.LT.AND P0, PT, RZ, c[0x0][0x180], PT, !PT; /* 0x00006000ff007a0c */ + /* 0x000fd00003f012f0 */ + /*01a0*/ @P1 IADD3 R5, R5, 0x1, RZ; /* 0x0000000105051810 */ + /* 0x000fe20007ffe0ff */ + /*01b0*/ IMAD R28, R28, c[0x0][0x4], R7; /* 0x000001001c1c7a24 */ + /* 0x001fc600078e0207 */ + /*01c0*/ SHF.L.U32 R29, R5, 0x4, RZ; /* 0x00000004051d7819 */ + /* 0x000fe200000006ff */ + /*01d0*/ IMAD.U32 R28, R28, 0x10, RZ; /* 0x000000101c1c7824 */ + /* 0x000fc600078e00ff */ + /*01e0*/ ISETP.LT.AND P0, PT, R29, c[0x0][0x178], P0, !PT; /* 0x00005e001d007a0c */ + /* 0x000fc800007012f0 */ + /*01f0*/ ISETP.LT.AND P0, PT, R28, c[0x0][0x17c], P0, !PT; /* 0x00005f001c007a0c */ + /* 0x000fe200007012f0 */ + /*0200*/ BSSY B0, 0x5b0; /* 0x000003a000007945 */ + /* 0x000fe20003800000 */ + /*0210*/ IMAD.U32 R7, RZ, RZ, RZ; /* 0x000000ffff077224 */ + /* 0x000fe200078e00ff */ + /*0220*/ MOV R11, RZ; /* 0x000000ff000b7202 */ + /* 0x000fe20000000f00 */ + /*0230*/ IMAD.U32 R6, RZ, RZ, RZ; /* 0x000000ffff067224 */ + /* 0x000fe400078e00ff */ + /*0240*/ IMAD.U32 R5, RZ, RZ, RZ; /* 0x000000ffff057224 */ + /* 0x000fe400078e00ff */ + /*0250*/ IMAD.U32 R4, RZ, RZ, RZ; /* 0x000000ffff047224 */ + /* 0x000fe400078e00ff */ + /*0260*/ IMAD.U32 R10, RZ, RZ, RZ; /* 0x000000ffff0a7224 */ + /* 0x000fc400078e00ff */ + /*0270*/ IMAD.U32 R9, RZ, RZ, RZ; /* 0x000000ffff097224 */ + /* 0x000fe400078e00ff */ + /*0280*/ IMAD.U32 R8, RZ, RZ, RZ; /* 0x000000ffff087224 */ + /* 0x000fe200078e00ff */ + /*0290*/ @!P0 BRA 0x5a0; /* 0x0000030000008947 */ + /* 0x000fee0003800000 */ + /*02a0*/ S2R R6, SR_LANEID; /* 0x0000000000067919 */ + /* 0x000e220000000000 */ + /*02b0*/ IMAD.U32 R10, RZ, RZ, 0x2; /* 0x00000002ff0a7424 */ + /* 0x000fc800078e00ff */ + /*02c0*/ IMAD.WIDE R16, R29, R10, c[0x0][0x160]; /* 0x000058001d107625 */ + /* 0x000fe200078e020a */ + /*02d0*/ SHF.R.U32.HI R4, RZ, 0x2, R6; /* 0x00000002ff047819 */ + /* 0x001fc80000011606 */ + /*02e0*/ LOP3.LUT R5, R4, 0x3, RZ, 0xc0, !PT; /* 0x0000000304057812 */ + /* 0x000fe400078ec0ff */ + /*02f0*/ LOP3.LUT R4, R6, 0x3, RZ, 0xc0, !PT; /* 0x0000000306047812 */ + /* 0x000fe400078ec0ff */ + /*0300*/ LOP3.LUT R7, R5, 0x1, RZ, 0xc0, !PT; /* 0x0000000105077812 */ + /* 0x000fe400078ec0ff */ + /*0310*/ SHF.R.U32.HI R6, RZ, 0x4, R6; /* 0x00000004ff067819 */ + /* 0x000fe40000011606 */ + /*0320*/ SHF.R.U32.HI R5, RZ, 0x1, R5; /* 0x00000001ff057819 */ + /* 0x000fe20000011605 */ + /*0330*/ IMAD R7, R7, 0x8, R4; /* 0x0000000807077824 */ + /* 0x000fe200078e0204 */ + /*0340*/ LOP3.LUT R6, R6, 0x1, RZ, 0xc0, !PT; /* 0x0000000106067812 */ + /* 0x000fc400078ec0ff */ + /*0350*/ LEA R4, R5, R4, 0x3; /* 0x0000000405047211 */ + /* 0x000fe200078e18ff */ + /*0360*/ IMAD.WIDE R12, R28, R10, c[0x0][0x168]; /* 0x00005a001c0c7625 */ + /* 0x000fe400078e020a */ + /*0370*/ IMAD R7, R6.reuse, 0x4, R7; /* 0x0000000406077824 */ + /* 0x040fe400078e0207 */ + /*0380*/ IMAD R4, R6, 0x4, R4; /* 0x0000000406047824 */ + /* 0x000fe400078e0204 */ + /*0390*/ IMAD.U32 R7, R7, 0x2, RZ; /* 0x0000000207077824 */ + /* 0x000fe400078e00ff */ + /*03a0*/ IMAD.U32 R5, R4, 0x2, RZ; /* 0x0000000204057824 */ + /* 0x000fe400078e00ff */ + /*03b0*/ IMAD.WIDE.U32 R16, R7, c[0x0][0x178], R16; /* 0x00005e0007107a25 */ + /* 0x000fc400078e0010 */ + /*03c0*/ IMAD.WIDE.U32 R12, R5, c[0x0][0x180], R12; /* 0x00006000050c7a25 */ + /* 0x000fd000078e000c */ + /*03d0*/ LD.E.128.SYS R24, [R16]; /* 0x0000000010187980 */ + /* 0x000064000010ed00 */ + /*03e0*/ LD.E.128.SYS R20, [R12]; /* 0x000000000c147980 */ + /* 0x000462000010ed00 */ + /*03f0*/ LD.E.128.SYS R16, [R16+0x10]; /* 0x0000001010107980 */ + /* 0x001e22000010ed00 */ + /*0400*/ LD.E.128.SYS R12, [R12+0x10]; /* 0x000000100c0c7980 */ + /* 0x004e22000010ed00 */ + /*0410*/ IMAD.U32 R8, RZ, RZ, RZ; /* 0x000000ffff087224 */ + /* 0x000fe200078e00ff */ + /*0420*/ MOV R9, RZ; /* 0x000000ff00097202 */ + /* 0x000fe20000000f00 */ + /*0430*/ IMAD.U32 R10, RZ, RZ, RZ; /* 0x000000ffff0a7224 */ + /* 0x000fe400078e00ff */ + /*0440*/ IMAD.U32 R11, RZ, RZ, RZ; /* 0x000000ffff0b7224 */ + /* 0x000fe200078e00ff */ + /*0450*/ MOV R7, RZ; /* 0x000000ff00077202 */ + /* 0x000fe20000000f00 */ + /*0460*/ IMAD.U32 R4, RZ, RZ, RZ; /* 0x000000ffff047224 */ + /* 0x000fc400078e00ff */ + /*0470*/ IMAD.U32 R5, RZ, RZ, RZ; /* 0x000000ffff057224 */ + /* 0x000fe400078e00ff */ + /*0480*/ IMAD.U32 R6, RZ, RZ, RZ; /* 0x000000ffff067224 */ + /* 0x000fe200078e00ff */ + /*0490*/ WARPSYNC 0xffffffff; /* 0xffffffff00007948 */ + /* 0x000fe20003800000 */ + /*04a0*/ HMMA.884.F32.F32.STEP0 R8, R24.reuse, R20.reuse.T, R8; /* 0x0000001418087236 */ + /* 0x0c22640000005408 */ + /*04b0*/ HMMA.884.F32.F32.STEP1 R10, R24.reuse, R20.reuse.T, R10; /* 0x00000014180a7236 */ + /* 0x0c04a4000000d40a */ + /*04c0*/ HMMA.884.F32.F32.STEP2 R4, R24.reuse, R20.reuse.T, R4; /* 0x0000001418047236 */ + /* 0x0c06e40000015404 */ + /*04d0*/ HMMA.884.F32.F32.STEP3 R6, R24, R20.T, R6; /* 0x0000001418067236 */ + /* 0x000928000001d406 */ + /*04e0*/ HMMA.884.F32.F32.STEP0 R8, R26.reuse, R22.reuse.T, R8; /* 0x000000161a087236 */ + /* 0x0c20240000005408 */ + /*04f0*/ HMMA.884.F32.F32.STEP1 R10, R26.reuse, R22.reuse.T, R10; /* 0x000000161a0a7236 */ + /* 0x0c4264000000d40a */ + /*0500*/ HMMA.884.F32.F32.STEP2 R4, R26.reuse, R22.reuse.T, R4; /* 0x000000161a047236 */ + /* 0x0c84a40000015404 */ + /*0510*/ HMMA.884.F32.F32.STEP3 R6, R26, R22.T, R6; /* 0x000000161a067236 */ + /* 0x0106e8000001d406 */ + /*0520*/ HMMA.884.F32.F32.STEP0 R8, R16.reuse, R12.reuse.T, R8; /* 0x0000000c10087236 */ + /* 0x0c10240000005408 */ + /*0530*/ HMMA.884.F32.F32.STEP1 R10, R16.reuse, R12.reuse.T, R10; /* 0x0000000c100a7236 */ + /* 0x0c2264000000d40a */ + /*0540*/ HMMA.884.F32.F32.STEP2 R4, R16.reuse, R12.reuse.T, R4; /* 0x0000000c10047236 */ + /* 0x0c44a40000015404 */ + /*0550*/ HMMA.884.F32.F32.STEP3 R6, R16, R12.T, R6; /* 0x0000000c10067236 */ + /* 0x0086e8000001d406 */ + /*0560*/ HMMA.884.F32.F32.STEP0 R8, R18.reuse, R14.reuse.T, R8; /* 0x0000000e12087236 */ + /* 0x0c10240000005408 */ + /*0570*/ HMMA.884.F32.F32.STEP1 R10, R18.reuse, R14.reuse.T, R10; /* 0x0000000e120a7236 */ + /* 0x0c2024000000d40a */ + /*0580*/ HMMA.884.F32.F32.STEP2 R4, R18.reuse, R14.reuse.T, R4; /* 0x0000000e12047236 */ + /* 0x0c40240000015404 */ + /*0590*/ HMMA.884.F32.F32.STEP3 R6, R18, R14.T, R6; /* 0x0000000e12067236 */ + /* 0x008034000001d406 */ + /*05a0*/ BSYNC B0; /* 0x0000000000007941 */ + /* 0x001fea0003800000 */ + /*05b0*/ S2R R12, SR_LANEID; /* 0x00000000000c7919 */ + /* 0x000e220000000000 */ + /*05c0*/ IMAD R28, R28, c[0x0][0x178], RZ; /* 0x00005e001c1c7a24 */ + /* 0x000fe200078e02ff */ + /*05d0*/ SHF.R.U32.HI R14, RZ, 0x4, R12; /* 0x00000004ff0e7819 */ + /* 0x001fc4000001160c */ + /*05e0*/ SHF.R.U32.HI R13, RZ, 0x2, R12; /* 0x00000002ff0d7819 */ + /* 0x000fe4000001160c */ + /*05f0*/ LOP3.LUT R12, R12, 0x3, RZ, 0xc0, !PT; /* 0x000000030c0c7812 */ + /* 0x000fe400078ec0ff */ + /*0600*/ LOP3.LUT R14, R14, 0x1, RZ, 0xc0, !PT; /* 0x000000010e0e7812 */ + /* 0x000fe400078ec0ff */ + /*0610*/ LOP3.LUT R13, R13, 0x3, RZ, 0xc0, !PT; /* 0x000000030d0d7812 */ + /* 0x000fc600078ec0ff */ + /*0620*/ IMAD R12, R14, 0x4, R12; /* 0x000000040e0c7824 */ + /* 0x000fe200078e020c */ + /*0630*/ LOP3.LUT R15, R13, 0x1, RZ, 0xc0, !PT; /* 0x000000010d0f7812 */ + /* 0x000fe400078ec0ff */ + /*0640*/ SHF.R.U32.HI R16, RZ, 0x1, R13; /* 0x00000001ff107819 */ + /* 0x000fe4000001160d */ + /*0650*/ LOP3.LUT R13, R12.reuse, 0x5, RZ, 0xc0, !PT; /* 0x000000050c0d7812 */ + /* 0x040fe400078ec0ff */ + /*0660*/ LOP3.LUT R14, R12, 0x2, RZ, 0xc0, !PT; /* 0x000000020c0e7812 */ + /* 0x000fc600078ec0ff */ + /*0670*/ IMAD R12, R15, 0x8, R13; /* 0x000000080f0c7824 */ + /* 0x000fe200078e020d */ + /*0680*/ SHF.R.S32.HI R15, RZ, 0x1f, R29; /* 0x0000001fff0f7819 */ + /* 0x000fe2000001141d */ + /*0690*/ IMAD R14, R16, 0x8, R14; /* 0x00000008100e7824 */ + /* 0x000fe200078e020e */ + /*06a0*/ IADD3 R17, P0, R29, R28, RZ; /* 0x0000001c1d117210 */ + /* 0x000fe20007f1e0ff */ + /*06b0*/ IMAD.U32 R13, RZ, RZ, RZ; /* 0x000000ffff0d7224 */ + /* 0x000fc600078e00ff */ + /*06c0*/ LEA.HI.X.SX32 R15, R28, R15, 0x1, P0; /* 0x0000000f1c0f7211 */ + /* 0x000fe200000f0eff */ + /*06d0*/ IMAD.WIDE.U32 R12, R14, c[0x0][0x178], R12; /* 0x00005e000e0c7a25 */ + /* 0x000fe200078e000c */ + /*06e0*/ LEA R14, P0, R17, c[0x0][0x170], 0x2; /* 0x00005c00110e7a11 */ + /* 0x000fe200078010ff */ + /*06f0*/ IMAD.U32 R18, RZ, RZ, c[0x0][0x178]; /* 0x00005e00ff127624 */ + /* 0x000fc600078e00ff */ + /*0700*/ LEA.HI.X R17, R17, c[0x0][0x174], R15, 0x2, P0; /* 0x00005d0011117a11 */ + /* 0x000fe400000f140f */ + /*0710*/ LEA R16, P0, R12, R14, 0x2; /* 0x0000000e0c107211 */ + /* 0x000fe400078010ff */ + /*0720*/ SHF.L.U32 R14, R18, 0x2, RZ; /* 0x00000002120e7819 */ + /* 0x000fe400000006ff */ + /*0730*/ SHF.R.U32.HI R15, RZ, 0x1e, R18; /* 0x0000001eff0f7819 */ + /* 0x000fe40000011612 */ + /*0740*/ LEA.HI.X R13, R12, R17, R13, 0x2, P0; /* 0x000000110c0d7211 */ + /* 0x000fe400000f140d */ + /*0750*/ LEA R18, P1, R14, R16, 0x2; /* 0x000000100e127211 */ + /* 0x000fc400078210ff */ + /*0760*/ IADD3 R17, P0, R14.reuse, R16, RZ; /* 0x000000100e117210 */ + /* 0x040fe40007f1e0ff */ + /*0770*/ LEA.HI.X R19, R14.reuse, R13, R15, 0x2, P1; /* 0x0000000d0e137211 */ + /* 0x040fe400008f140f */ + /*0780*/ IADD3 R21, P1, R14, R18, RZ; /* 0x000000120e157210 */ + /* 0x000fe20007f3e0ff */ + /*0790*/ IMAD.X R20, R15.reuse, 0x1, R13, P0; /* 0x000000010f147824 */ + /* 0x040fe400000e060d */ + /*07a0*/ IMAD.U32 R12, RZ, RZ, R16; /* 0x000000ffff0c7224 */ + /* 0x000fe400078e0010 */ + /*07b0*/ IMAD.X R22, R15, 0x1, R19, P1; /* 0x000000010f167824 */ + /* 0x000fe400008e0613 */ + /*07c0*/ IMAD.U32 R14, RZ, RZ, R17; /* 0x000000ffff0e7224 */ + /* 0x000fc400078e0011 */ + /*07d0*/ IMAD.U32 R15, RZ, RZ, R20; /* 0x000000ffff0f7224 */ + /* 0x000fe200078e0014 */ + /*07e0*/ MOV R16, R21; /* 0x0000001500107202 */ + /* 0x000fe20000000f00 */ + /*07f0*/ IMAD.U32 R17, RZ, RZ, R22; /* 0x000000ffff117224 */ + /* 0x000fe200078e0016 */ + /*0800*/ ST.E.SYS [R12], R8; /* 0x000000000c007385 */ + /* 0x0001e2000010e908 */ + /*0810*/ ST.E.SYS [R12+0x8], R10; /* 0x000000080c007385 */ + /* 0x0003e8000010e90a */ + /*0820*/ ST.E.SYS [R14], R9; /* 0x000000000e007385 */ + /* 0x0003e2000010e909 */ + /*0830*/ ST.E.SYS [R14+0x8], R11; /* 0x000000080e007385 */ + /* 0x0003e2000010e90b */ + /*0840*/ ST.E.SYS [R18], R4; /* 0x0000000012007385 */ + /* 0x0003e2000010e904 */ + /*0850*/ ST.E.SYS [R18+0x8], R6; /* 0x0000000812007385 */ + /* 0x0003e2000010e906 */ + /*0860*/ ST.E.SYS [R16], R5; /* 0x0000000010007385 */ + /* 0x0003e2000010e905 */ + /*0870*/ ST.E.SYS [R16+0x8], R7; /* 0x0000000810007385 */ + /* 0x0003e2000010e907 */ + /*0880*/ WARPSYNC 0xffffffff; /* 0xffffffff00007948 */ + /* 0x000fe20003800000 */ + /*0890*/ IADD3 R8, R2, -c[0x0][0x20], RZ; /* 0x8000080002087a10 */ + /* 0x001fd00007ffe0ff */ + /*08a0*/ CS2R.32 R4, SR_CLOCKLO; /* 0x0000000000047805 */ + /* 0x002fd00000005000 */ + /*08b0*/ IMAD R3, R4, 0x1, -R3; /* 0x0000000104037824 */ + /* 0x000fd000078e0a03 */ + /*08c0*/ STL [R8], R3; /* 0x0000000308007387 */ + /* 0x0001e20000100800 */ + /*08d0*/ IMAD.U32 R6, RZ, RZ, R2; /* 0x000000ffff067224 */ + /* 0x000fe200078e0002 */ + /*08e0*/ MOV R4, 0x0; /* 0x0000000000047802 */ + /* 0x000fe20000000f00 */ + /*08f0*/ IMAD.U32 R7, RZ, RZ, R0; /* 0x000000ffff077224 */ + /* 0x000fe200078e0000 */ + /*0900*/ MOV R5, 0x0; /* 0x0000000000057802 */ + /* 0x000fe40000000f00 */ + /*0910*/ MOV R20, 0x0; /* 0x0000000000147802 */ + /* 0x000fe40000000f00 */ + /*0920*/ MOV R21, 0x0; /* 0x0000000000157802 */ + /* 0x000fd00000000f00 */ + /*0930*/ CALL.ABS.NOINC 0x0; /* 0x0000000000007943 */ + /* 0x001fea0003c00000 */ + /*0940*/ EXIT; /* 0x000000000000794d */ + /* 0x000fea0003800000 */ + /*0950*/ BRA 0x950; /* 0xfffffff000007947 */ + /* 0x000fc0000383ffff */ + /*0960*/ NOP; /* 0x0000000000007918 */ + /* 0x000fc00000000000 */ + /*0970*/ NOP; /* 0x0000000000007918 */ + /* 0x000fc00000000000 */ + ............................................. + + + +Fatbin ptx code: +================ +arch = sm_70 +code version = [6,0] +producer = cuda +host = linux +compile_size = 64bit +compressed + + + + + + + + +.version 6.0 +.target sm_70 +.address_size 64 + + +.extern .func (.param .b32 func_retval0) vprintf +( +.param .b64 vprintf_param_0, +.param .b64 vprintf_param_1 +) +; +.global .align 16 .b8 $str[9] = {99, 108, 111, 99, 107, 61, 37, 100, 0}; + +.visible .entry _Z12wmma_exampleP6__halfS0_Pfiiiff( +.param .u64 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_0, +.param .u64 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_1, +.param .u64 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_2, +.param .u32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_3, +.param .u32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_4, +.param .u32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_5, +.param .f32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_6, +.param .f32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_7 +) +{ +.local .align 8 .b8 __local_depot0[8]; +.reg .b64 %SP; +.reg .b64 %SPL; +.reg .pred %p<6>; +.reg .f32 %f<34>; +.reg .b32 %r<38>; +.reg .b64 %rd<18>; + + +mov.u64 %rd17, __local_depot0; +cvta.local.u64 %SP, %rd17; +ld.param.u64 %rd1, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_0]; +ld.param.u64 %rd2, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_1]; +ld.param.u64 %rd3, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_2]; +ld.param.u32 %r4, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_3]; +ld.param.u32 %r7, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_4]; +ld.param.u32 %r5, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_5]; + + mov.u32 %r6, %clock; + + mov.u32 %r8, %ntid.x; +mov.u32 %r9, %ctaid.x; +mov.u32 %r10, %tid.x; +mad.lo.s32 %r11, %r8, %r9, %r10; +mov.u32 %r12, WARP_SZ; +div.u32 %r13, %r11, %r12; +mov.u32 %r14, %ntid.y; +mov.u32 %r15, %ctaid.y; +mov.u32 %r16, %tid.y; +mad.lo.s32 %r17, %r14, %r15, %r16; +shl.b32 %r2, %r13, 4; +shl.b32 %r3, %r17, 4; +setp.lt.s32 %p1, %r2, %r4; +setp.gt.s32 %p2, %r5, 0; +and.pred %p3, %p1, %p2; +setp.lt.s32 %p4, %r3, %r7; +and.pred %p5, %p3, %p4; +mov.f32 %f26, 0f00000000; +mov.f32 %f27, %f26; +mov.f32 %f28, %f26; +mov.f32 %f29, %f26; +mov.f32 %f30, %f26; +mov.f32 %f31, %f26; +mov.f32 %f32, %f26; +mov.f32 %f33, %f26; +@!%p5 bra BB0_2; +bra.uni BB0_1; + +BB0_1: +mul.wide.s32 %rd4, %r2, 2; +add.s64 %rd5, %rd1, %rd4; +wmma.load.a.sync.row.m16n16k16.f16 {%r18, %r19, %r20, %r21, %r22, %r23, %r24, %r25}, [%rd5], %r4; +mul.wide.s32 %rd6, %r3, 2; +add.s64 %rd7, %rd2, %rd6; +wmma.load.b.sync.col.m16n16k16.f16 {%r26, %r27, %r28, %r29, %r30, %r31, %r32, %r33}, [%rd7], %r5; +mov.f32 %f25, 0f00000000; +wmma.mma.sync.row.col.m16n16k16.f32.f32 {%f33, %f32, %f31, %f30, %f29, %f28, %f27, %f26}, {%r18, %r19, %r20, %r21, %r22, %r23, %r24, %r25}, {%r26, %r27, %r28, %r29, %r30, %r31, %r32, %r33}, {%f25, %f25, %f25, %f25, %f25, %f25, %f25, %f25}; + +BB0_2: +add.u64 %rd8, %SP, 0; +cvta.to.local.u64 %rd9, %rd8; +mul.lo.s32 %r35, %r3, %r4; +cvt.s64.s32 %rd10, %r35; +cvt.s64.s32 %rd11, %r2; +add.s64 %rd12, %rd10, %rd11; +shl.b64 %rd13, %rd12, 2; +add.s64 %rd14, %rd3, %rd13; +wmma.store.d.sync.col.m16n16k16.f32 [%rd14], {%f33, %f32, %f31, %f30, %f29, %f28, %f27, %f26}, %r4; + + mov.u32 %r34, %clock; + + sub.s32 %r36, %r34, %r6; +st.local.u32 [%rd9], %r36; +mov.u64 %rd15, $str; +cvta.global.u64 %rd16, %rd15; + + { +.reg .b32 temp_param_reg; + + .param .b64 param0; +st.param.b64 [param0+0], %rd16; +.param .b64 param1; +st.param.b64 [param1+0], %rd8; +.param .b32 retval0; +call.uni (retval0), +vprintf, +( +param0, +param1 +); +ld.param.b32 %r37, [retval0+0]; + + + } + ret; +} + + +.visible .entry _Z17convertFp32ToFp16P6__halfPfi( +.param .u64 _Z17convertFp32ToFp16P6__halfPfi_param_0, +.param .u64 _Z17convertFp32ToFp16P6__halfPfi_param_1, +.param .u32 _Z17convertFp32ToFp16P6__halfPfi_param_2 +) +{ +.reg .pred %p<2>; +.reg .b16 %rs<2>; +.reg .f32 %f<2>; +.reg .b32 %r<6>; +.reg .b64 %rd<9>; + + +ld.param.u64 %rd1, [_Z17convertFp32ToFp16P6__halfPfi_param_0]; +ld.param.u64 %rd2, [_Z17convertFp32ToFp16P6__halfPfi_param_1]; +ld.param.u32 %r2, [_Z17convertFp32ToFp16P6__halfPfi_param_2]; +mov.u32 %r3, %ntid.x; +mov.u32 %r4, %ctaid.x; +mov.u32 %r5, %tid.x; +mad.lo.s32 %r1, %r4, %r3, %r5; +setp.ge.s32 %p1, %r1, %r2; +@%p1 bra BB1_2; + +cvta.to.global.u64 %rd3, %rd2; +mul.wide.s32 %rd4, %r1, 4; +add.s64 %rd5, %rd3, %rd4; +ld.global.f32 %f1, [%rd5]; + + { cvt.rn.f16.f32 %rs1, %f1;} + + + cvta.to.global.u64 %rd6, %rd1; +mul.wide.s32 %rd7, %r1, 2; +add.s64 %rd8, %rd6, %rd7; +st.global.u16 [%rd8], %rs1; + +BB1_2: +ret; +} + + diff --git a/cuda-kernels/config_fermi_islip.icnt b/cuda-kernels/config_fermi_islip.icnt new file mode 100755 index 0000000..a788090 --- /dev/null +++ b/cuda-kernels/config_fermi_islip.icnt @@ -0,0 +1,70 @@ +//21*1 fly with 32 flits per packet under gpgpusim injection mode +use_map = 0; +flit_size = 32; + +// currently we do not use this, see subnets below +network_count = 2; + +// Topology +topology = fly; +k = 62; +n = 1; + +// Routing + +routing_function = dest_tag; + +// Flow control + +num_vcs = 1; +vc_buf_size = 8; + +wait_for_tail_credit = 0; + +// Router architecture + +vc_allocator = islip; //separable_input_first; +sw_allocator = islip; //separable_input_first; +alloc_iters = 1; + +credit_delay = 0; +routing_delay = 0; +vc_alloc_delay = 1; +sw_alloc_delay = 1; + +input_speedup = 2; +output_speedup = 1; +internal_speedup = 1.0; + +// Traffic, GPGPU-Sim does not use this + +traffic = uniform; +packet_size ={{1,2,3,4},{10,20}}; +packet_size_rate={{1,1,1,1},{2,1}}; + +// Simulation - Don't change + +sim_type = gpgpusim; +//sim_type = latency; +injection_rate = 0.1; + +subnets = 2; + +// Always use read and write no matter following line +//use_read_write = 1; + + +read_request_subnet = 0; +read_reply_subnet = 1; +write_request_subnet = 0; +write_reply_subnet = 1; + +read_request_begin_vc = 0; +read_request_end_vc = 0; +write_request_begin_vc = 0; +write_request_end_vc = 0; +read_reply_begin_vc = 0; +read_reply_end_vc = 0; +write_reply_begin_vc = 0; +write_reply_end_vc = 0; + diff --git a/cuda-kernels/gpgpu_inst_stats.txt b/cuda-kernels/gpgpu_inst_stats.txt new file mode 100755 index 0000000..acb1839 --- /dev/null +++ b/cuda-kernels/gpgpu_inst_stats.txt @@ -0,0 +1 @@ +kernel line : count latency dram_traffic smem_bk_conflicts smem_warp gmem_access_generated gmem_warp exposed_latency warp_divergence diff --git a/cuda-kernels/gpgpusim.config b/cuda-kernels/gpgpusim.config new file mode 100755 index 0000000..306d7f9 --- /dev/null +++ b/cuda-kernels/gpgpusim.config @@ -0,0 +1,149 @@ +# This config models the Pascal GP102 (GeForceGTX 1080Ti) + +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 70 + +# SASS execution (only supported with CUDA >= 4.0) +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 + +# high level architecture configuration +-gpgpu_n_clusters 40 +-gpgpu_n_cores_per_cluster 1 +-gpgpu_n_mem 11 +-gpgpu_n_sub_partition_per_mchannel 2 + +# Pascal clock domains +#-gpgpu_clock_domains ::: +# Pascal NVIDIA TITAN X clock domains are adopted from +# https://en.wikipedia.org/wiki/GeForce_10_series +-gpgpu_clock_domains 1481.0:2962.0:1481.0:2750.0 + +# shader core pipeline config +-gpgpu_shader_registers 65536 + +# This implies a maximum of 64 warps/SM +-gpgpu_shader_core_pipeline 2048:32 +-gpgpu_shader_cta 32 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB +## Pascal GP102 has 4 SP SIMD units and 1 SFU unit +## we need to scale the number of pipeline registers to be equal to the number of SP units +-gpgpu_pipeline_widths 4,1,1,4,1,1,6 +-gpgpu_num_sp_units 4 +-gpgpu_num_sfu_units 1 + +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +# SFU is 32-width in pascal, then dp units initiation is 1 cycle +-ptx_opcode_latency_int 4,13,4,5,145,16,4 +-ptx_opcode_initiation_int 1,2,2,2,8,16,4 +-ptx_opcode_latency_fp 4,13,4,5,39 +-ptx_opcode_initiation_fp 1,2,1,1,4 +-ptx_opcode_latency_dp 8,19,8,8,330 +-ptx_opcode_initiation_dp 1,2,1,1,130 + +# ::,::::,::,:** +# ** Optional parameter - Required when mshr_type==Texture Fifo +# Note: Hashing set index function (H) only applies to a set size of 32 or 64. +# Pascal GP102 has 96KB Shared memory +# Pascal GP102 has 64KB L1 cache +# The default is to disable the L1 cache, unless cache modifieres is used +-gpgpu_cache:dl1 64:128:6,L:L:m:N:H,A:128:8,8 +-gpgpu_shmem_size 98304 +-gmem_skip_L1D 1 + +# 64 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 3MB L2 cache +-gpgpu_cache:dl2 64:128:16,L:B:m:W:L,A:1024:1024,4:0,32 # used to be 128:4 +-gpgpu_cache:dl2_texture_only 0 + +# 4 KB Inst. +-gpgpu_cache:il1 8:128:4,L:R:f:N:L,A:2:48,4 +# 48 KB Tex +-gpgpu_tex_cache:l1 16:128:24,L:R:m:N:L,F:128:4,128:2 +# 12 KB Const +-gpgpu_const_cache:l1 128:64:2,L:R:f:N:L,A:2:64,4 + +# enable operand collector +## larger operand collectors and reg_banks are needed for the 4 warp schedulers and 4 SIMD units +-gpgpu_operand_collector_num_units_sp 20 +-gpgpu_operand_collector_num_units_sfu 4 +-gpgpu_operand_collector_num_units_mem 8 +-gpgpu_operand_collector_num_in_ports_sp 4 +-gpgpu_operand_collector_num_out_ports_sp 4 +-gpgpu_operand_collector_num_in_ports_sfu 1 +-gpgpu_operand_collector_num_out_ports_sfu 1 +-gpgpu_operand_collector_num_in_ports_mem 1 +-gpgpu_operand_collector_num_out_ports_mem 1 +# gpgpu_num_reg_banks should be increased to 32, but it gives an error! +-gpgpu_num_reg_banks 32 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 + +## In Pascal, a warp scheduler can issue 2 insts per cycle +-gpgpu_max_insn_issue_per_warp 2 + +# interconnection +-network_mode 1 +-inter_config_file config_fermi_islip.icnt + +# memory partition latency config +-rop_latency 120 +-dram_latency 100 + +# dram model config +-gpgpu_dram_scheduler 1 +# The DRAM return queue and the scheduler queue together should provide buffer +# to sustain the memory level parallelism to tolerate DRAM latency +# To allow 100% DRAM utility, there should at least be enough buffer to sustain +# the minimum DRAM latency (100 core cycles). I.e. +# Total buffer space required = 100 x 924MHz / 700MHz = 132 +-gpgpu_frfcfs_dram_sched_queue_size 64 +-gpgpu_dram_return_queue_size 116 + +# for NVIDIA GeForceGTX 1080Ti, bus width is 352bits (11 DRAM chips x 32 bits) +# 11 memory paritions, 4 bytes (1 DRAM chip) per memory partition +# the atom size of GDDR5X (the smallest read request) is 32 bytes +-gpgpu_n_mem_per_ctrlr 1 +-gpgpu_dram_buswidth 4 +-gpgpu_dram_burst_length 8 +-dram_data_command_freq_ratio 4 # GDDR5X is QDR +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS + +# Use the same GDDR5 timing from hynix H5GQ1H24AFR +# disable bank groups for now, set nbkgrp to 1 and tCCDL and tRTPL to 0 +-gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=6:RCD=12:RAS=28:RP=12:RC=40: + CL=12:WL=4:CDLR=5:WR=12:nbkgrp=1:CCDL=0:RTPL=0" + +# Pascal has four schedulers per core +-gpgpu_num_sched_per_core 2 +# Two Level Scheduler with active and pending pools +#-gpgpu_scheduler two_level_active:6:0:1 +# Loose round robbin scheduler +#-gpgpu_scheduler lrr +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 +-visualizer_enabled 0 + +# power model configs +-power_simulation_enabled 1 +-gpuwattch_xml_file gpuwattch_gtx1080Ti.xml + +# tracing functionality +#-trace_enabled 1 +#-trace_components WARP_SCHEDULER,SCOREBOARD +#-trace_sampling_core 0 + diff --git a/cuda-kernels/gpgpusim_power_report__Sun-May-27-14-17-34-2018.log b/cuda-kernels/gpgpusim_power_report__Sun-May-27-14-17-34-2018.log new file mode 100644 index 0000000..f754f0c --- /dev/null +++ b/cuda-kernels/gpgpusim_power_report__Sun-May-27-14-17-34-2018.log @@ -0,0 +1,324 @@ +kernel_name = +kernel_launch_uid = + +Kernel Average Power Data: +kernel_avg_power = 0 +gpu_avg_IBP, = -nan +gpu_avg_ICP, = -nan +gpu_avg_DCP, = -nan +gpu_avg_TCP, = -nan +gpu_avg_CCP, = -nan +gpu_avg_SHRDP, = -nan +gpu_avg_RFP, = -nan +gpu_avg_SPP, = -nan +gpu_avg_SFUP, = -nan +gpu_avg_FPUP, = -nan +gpu_avg_SCHEDP, = -nan +gpu_avg_L2CP, = -nan +gpu_avg_MCP, = -nan +gpu_avg_NOCP, = -nan +gpu_avg_DRAMP, = -nan +gpu_avg_PIPEP, = -nan +gpu_avg_IDLE_COREP, = -nan +gpu_avg_CONST_DYNAMICP = -nan +gpu_avg_TOT_INST, = -nan +gpu_avg_FP_INT, = -nan +gpu_avg_IC_H, = -nan +gpu_avg_IC_M, = -nan +gpu_avg_DC_RH, = -nan +gpu_avg_DC_RM, = -nan +gpu_avg_DC_WH, = -nan +gpu_avg_DC_WM, = -nan +gpu_avg_TC_H, = -nan +gpu_avg_TC_M, = -nan +gpu_avg_CC_H, = -nan +gpu_avg_CC_M, = -nan +gpu_avg_SHRD_ACC, = -nan +gpu_avg_REG_RD, = -nan +gpu_avg_REG_WR, = -nan +gpu_avg_NON_REG_OPs, = -nan +gpu_avg_SP_ACC, = -nan +gpu_avg_SFU_ACC, = -nan +gpu_avg_FPU_ACC, = -nan +gpu_avg_MEM_RD, = -nan +gpu_avg_MEM_WR, = -nan +gpu_avg_MEM_PRE, = -nan +gpu_avg_L2_RH, = -nan +gpu_avg_L2_RM, = -nan +gpu_avg_L2_WH, = -nan +gpu_avg_L2_WM, = -nan +gpu_avg_NOC_A, = -nan +gpu_avg_PIPE_A, = -nan +gpu_avg_IDLE_CORE_N, = -nan +gpu_avg_CONST_DYNAMICN = -nan + +Kernel Maximum Power Data: +kernel_max_power = 0 +gpu_max_IBP, = 0 +gpu_max_ICP, = 0 +gpu_max_DCP, = 0 +gpu_max_TCP, = 0 +gpu_max_CCP, = 0 +gpu_max_SHRDP, = 0 +gpu_max_RFP, = 0 +gpu_max_SPP, = 0 +gpu_max_SFUP, = 0 +gpu_max_FPUP, = 0 +gpu_max_SCHEDP, = 0 +gpu_max_L2CP, = 0 +gpu_max_MCP, = 0 +gpu_max_NOCP, = 0 +gpu_max_DRAMP, = 0 +gpu_max_PIPEP, = 0 +gpu_max_IDLE_COREP, = 0 +gpu_max_CONST_DYNAMICP = 0 +gpu_max_TOT_INST, = 0 +gpu_max_FP_INT, = 0 +gpu_max_IC_H, = 0 +gpu_max_IC_M, = 0 +gpu_max_DC_RH, = 0 +gpu_max_DC_RM, = 0 +gpu_max_DC_WH, = 0 +gpu_max_DC_WM, = 0 +gpu_max_TC_H, = 0 +gpu_max_TC_M, = 0 +gpu_max_CC_H, = 0 +gpu_max_CC_M, = 0 +gpu_max_SHRD_ACC, = 0 +gpu_max_REG_RD, = 0 +gpu_max_REG_WR, = 0 +gpu_max_NON_REG_OPs, = 0 +gpu_max_SP_ACC, = 0 +gpu_max_SFU_ACC, = 0 +gpu_max_FPU_ACC, = 0 +gpu_max_MEM_RD, = 0 +gpu_max_MEM_WR, = 0 +gpu_max_MEM_PRE, = 0 +gpu_max_L2_RH, = 0 +gpu_max_L2_RM, = 0 +gpu_max_L2_WH, = 0 +gpu_max_L2_WM, = 0 +gpu_max_NOC_A, = 0 +gpu_max_PIPE_A, = 0 +gpu_max_IDLE_CORE_N, = 0 +gpu_max_CONST_DYNAMICN = 0 + +Kernel Minimum Power Data: +kernel_min_power = 0 +gpu_min_IBP, = 0 +gpu_min_ICP, = 0 +gpu_min_DCP, = 0 +gpu_min_TCP, = 0 +gpu_min_CCP, = 0 +gpu_min_SHRDP, = 0 +gpu_min_RFP, = 0 +gpu_min_SPP, = 0 +gpu_min_SFUP, = 0 +gpu_min_FPUP, = 0 +gpu_min_SCHEDP, = 0 +gpu_min_L2CP, = 0 +gpu_min_MCP, = 0 +gpu_min_NOCP, = 0 +gpu_min_DRAMP, = 0 +gpu_min_PIPEP, = 0 +gpu_min_IDLE_COREP, = 0 +gpu_min_CONST_DYNAMICP = 0 +gpu_min_TOT_INST, = 0 +gpu_min_FP_INT, = 0 +gpu_min_IC_H, = 0 +gpu_min_IC_M, = 0 +gpu_min_DC_RH, = 0 +gpu_min_DC_RM, = 0 +gpu_min_DC_WH, = 0 +gpu_min_DC_WM, = 0 +gpu_min_TC_H, = 0 +gpu_min_TC_M, = 0 +gpu_min_CC_H, = 0 +gpu_min_CC_M, = 0 +gpu_min_SHRD_ACC, = 0 +gpu_min_REG_RD, = 0 +gpu_min_REG_WR, = 0 +gpu_min_NON_REG_OPs, = 0 +gpu_min_SP_ACC, = 0 +gpu_min_SFU_ACC, = 0 +gpu_min_FPU_ACC, = 0 +gpu_min_MEM_RD, = 0 +gpu_min_MEM_WR, = 0 +gpu_min_MEM_PRE, = 0 +gpu_min_L2_RH, = 0 +gpu_min_L2_RM, = 0 +gpu_min_L2_WH, = 0 +gpu_min_L2_WM, = 0 +gpu_min_NOC_A, = 0 +gpu_min_PIPE_A, = 0 +gpu_min_IDLE_CORE_N, = 0 +gpu_min_CONST_DYNAMICN = 0 + +Accumulative Power Statistics Over Previous Kernels: +gpu_tot_avg_power = -nan +gpu_tot_max_power = 0 +gpu_tot_min_power = 0 + + +kernel_name = +kernel_launch_uid = + +Kernel Average Power Data: +kernel_avg_power = 0 +gpu_avg_IBP, = -nan +gpu_avg_ICP, = -nan +gpu_avg_DCP, = -nan +gpu_avg_TCP, = -nan +gpu_avg_CCP, = -nan +gpu_avg_SHRDP, = -nan +gpu_avg_RFP, = -nan +gpu_avg_SPP, = -nan +gpu_avg_SFUP, = -nan +gpu_avg_FPUP, = -nan +gpu_avg_SCHEDP, = -nan +gpu_avg_L2CP, = -nan +gpu_avg_MCP, = -nan +gpu_avg_NOCP, = -nan +gpu_avg_DRAMP, = -nan +gpu_avg_PIPEP, = -nan +gpu_avg_IDLE_COREP, = -nan +gpu_avg_CONST_DYNAMICP = -nan +gpu_avg_TOT_INST, = -nan +gpu_avg_FP_INT, = -nan +gpu_avg_IC_H, = -nan +gpu_avg_IC_M, = -nan +gpu_avg_DC_RH, = -nan +gpu_avg_DC_RM, = -nan +gpu_avg_DC_WH, = -nan +gpu_avg_DC_WM, = -nan +gpu_avg_TC_H, = -nan +gpu_avg_TC_M, = -nan +gpu_avg_CC_H, = -nan +gpu_avg_CC_M, = -nan +gpu_avg_SHRD_ACC, = -nan +gpu_avg_REG_RD, = -nan +gpu_avg_REG_WR, = -nan +gpu_avg_NON_REG_OPs, = -nan +gpu_avg_SP_ACC, = -nan +gpu_avg_SFU_ACC, = -nan +gpu_avg_FPU_ACC, = -nan +gpu_avg_MEM_RD, = -nan +gpu_avg_MEM_WR, = -nan +gpu_avg_MEM_PRE, = -nan +gpu_avg_L2_RH, = -nan +gpu_avg_L2_RM, = -nan +gpu_avg_L2_WH, = -nan +gpu_avg_L2_WM, = -nan +gpu_avg_NOC_A, = -nan +gpu_avg_PIPE_A, = -nan +gpu_avg_IDLE_CORE_N, = -nan +gpu_avg_CONST_DYNAMICN = -nan + +Kernel Maximum Power Data: +kernel_max_power = 0 +gpu_max_IBP, = 0 +gpu_max_ICP, = 0 +gpu_max_DCP, = 0 +gpu_max_TCP, = 0 +gpu_max_CCP, = 0 +gpu_max_SHRDP, = 0 +gpu_max_RFP, = 0 +gpu_max_SPP, = 0 +gpu_max_SFUP, = 0 +gpu_max_FPUP, = 0 +gpu_max_SCHEDP, = 0 +gpu_max_L2CP, = 0 +gpu_max_MCP, = 0 +gpu_max_NOCP, = 0 +gpu_max_DRAMP, = 0 +gpu_max_PIPEP, = 0 +gpu_max_IDLE_COREP, = 0 +gpu_max_CONST_DYNAMICP = 0 +gpu_max_TOT_INST, = 0 +gpu_max_FP_INT, = 0 +gpu_max_IC_H, = 0 +gpu_max_IC_M, = 0 +gpu_max_DC_RH, = 0 +gpu_max_DC_RM, = 0 +gpu_max_DC_WH, = 0 +gpu_max_DC_WM, = 0 +gpu_max_TC_H, = 0 +gpu_max_TC_M, = 0 +gpu_max_CC_H, = 0 +gpu_max_CC_M, = 0 +gpu_max_SHRD_ACC, = 0 +gpu_max_REG_RD, = 0 +gpu_max_REG_WR, = 0 +gpu_max_NON_REG_OPs, = 0 +gpu_max_SP_ACC, = 0 +gpu_max_SFU_ACC, = 0 +gpu_max_FPU_ACC, = 0 +gpu_max_MEM_RD, = 0 +gpu_max_MEM_WR, = 0 +gpu_max_MEM_PRE, = 0 +gpu_max_L2_RH, = 0 +gpu_max_L2_RM, = 0 +gpu_max_L2_WH, = 0 +gpu_max_L2_WM, = 0 +gpu_max_NOC_A, = 0 +gpu_max_PIPE_A, = 0 +gpu_max_IDLE_CORE_N, = 0 +gpu_max_CONST_DYNAMICN = 0 + +Kernel Minimum Power Data: +kernel_min_power = 0 +gpu_min_IBP, = 0 +gpu_min_ICP, = 0 +gpu_min_DCP, = 0 +gpu_min_TCP, = 0 +gpu_min_CCP, = 0 +gpu_min_SHRDP, = 0 +gpu_min_RFP, = 0 +gpu_min_SPP, = 0 +gpu_min_SFUP, = 0 +gpu_min_FPUP, = 0 +gpu_min_SCHEDP, = 0 +gpu_min_L2CP, = 0 +gpu_min_MCP, = 0 +gpu_min_NOCP, = 0 +gpu_min_DRAMP, = 0 +gpu_min_PIPEP, = 0 +gpu_min_IDLE_COREP, = 0 +gpu_min_CONST_DYNAMICP = 0 +gpu_min_TOT_INST, = 0 +gpu_min_FP_INT, = 0 +gpu_min_IC_H, = 0 +gpu_min_IC_M, = 0 +gpu_min_DC_RH, = 0 +gpu_min_DC_RM, = 0 +gpu_min_DC_WH, = 0 +gpu_min_DC_WM, = 0 +gpu_min_TC_H, = 0 +gpu_min_TC_M, = 0 +gpu_min_CC_H, = 0 +gpu_min_CC_M, = 0 +gpu_min_SHRD_ACC, = 0 +gpu_min_REG_RD, = 0 +gpu_min_REG_WR, = 0 +gpu_min_NON_REG_OPs, = 0 +gpu_min_SP_ACC, = 0 +gpu_min_SFU_ACC, = 0 +gpu_min_FPU_ACC, = 0 +gpu_min_MEM_RD, = 0 +gpu_min_MEM_WR, = 0 +gpu_min_MEM_PRE, = 0 +gpu_min_L2_RH, = 0 +gpu_min_L2_RM, = 0 +gpu_min_L2_WH, = 0 +gpu_min_L2_WM, = 0 +gpu_min_NOC_A, = 0 +gpu_min_PIPE_A, = 0 +gpu_min_IDLE_CORE_N, = 0 +gpu_min_CONST_DYNAMICN = 0 + +Accumulative Power Statistics Over Previous Kernels: +gpu_tot_avg_power = -nan +gpu_tot_max_power = 0 +gpu_tot_min_power = 0 + + diff --git a/cuda-kernels/gpgpusim_power_report__Sun-May-27-14-17-47-2018.log b/cuda-kernels/gpgpusim_power_report__Sun-May-27-14-17-47-2018.log new file mode 100644 index 0000000..f754f0c --- /dev/null +++ b/cuda-kernels/gpgpusim_power_report__Sun-May-27-14-17-47-2018.log @@ -0,0 +1,324 @@ +kernel_name = +kernel_launch_uid = + +Kernel Average Power Data: +kernel_avg_power = 0 +gpu_avg_IBP, = -nan +gpu_avg_ICP, = -nan +gpu_avg_DCP, = -nan +gpu_avg_TCP, = -nan +gpu_avg_CCP, = -nan +gpu_avg_SHRDP, = -nan +gpu_avg_RFP, = -nan +gpu_avg_SPP, = -nan +gpu_avg_SFUP, = -nan +gpu_avg_FPUP, = -nan +gpu_avg_SCHEDP, = -nan +gpu_avg_L2CP, = -nan +gpu_avg_MCP, = -nan +gpu_avg_NOCP, = -nan +gpu_avg_DRAMP, = -nan +gpu_avg_PIPEP, = -nan +gpu_avg_IDLE_COREP, = -nan +gpu_avg_CONST_DYNAMICP = -nan +gpu_avg_TOT_INST, = -nan +gpu_avg_FP_INT, = -nan +gpu_avg_IC_H, = -nan +gpu_avg_IC_M, = -nan +gpu_avg_DC_RH, = -nan +gpu_avg_DC_RM, = -nan +gpu_avg_DC_WH, = -nan +gpu_avg_DC_WM, = -nan +gpu_avg_TC_H, = -nan +gpu_avg_TC_M, = -nan +gpu_avg_CC_H, = -nan +gpu_avg_CC_M, = -nan +gpu_avg_SHRD_ACC, = -nan +gpu_avg_REG_RD, = -nan +gpu_avg_REG_WR, = -nan +gpu_avg_NON_REG_OPs, = -nan +gpu_avg_SP_ACC, = -nan +gpu_avg_SFU_ACC, = -nan +gpu_avg_FPU_ACC, = -nan +gpu_avg_MEM_RD, = -nan +gpu_avg_MEM_WR, = -nan +gpu_avg_MEM_PRE, = -nan +gpu_avg_L2_RH, = -nan +gpu_avg_L2_RM, = -nan +gpu_avg_L2_WH, = -nan +gpu_avg_L2_WM, = -nan +gpu_avg_NOC_A, = -nan +gpu_avg_PIPE_A, = -nan +gpu_avg_IDLE_CORE_N, = -nan +gpu_avg_CONST_DYNAMICN = -nan + +Kernel Maximum Power Data: +kernel_max_power = 0 +gpu_max_IBP, = 0 +gpu_max_ICP, = 0 +gpu_max_DCP, = 0 +gpu_max_TCP, = 0 +gpu_max_CCP, = 0 +gpu_max_SHRDP, = 0 +gpu_max_RFP, = 0 +gpu_max_SPP, = 0 +gpu_max_SFUP, = 0 +gpu_max_FPUP, = 0 +gpu_max_SCHEDP, = 0 +gpu_max_L2CP, = 0 +gpu_max_MCP, = 0 +gpu_max_NOCP, = 0 +gpu_max_DRAMP, = 0 +gpu_max_PIPEP, = 0 +gpu_max_IDLE_COREP, = 0 +gpu_max_CONST_DYNAMICP = 0 +gpu_max_TOT_INST, = 0 +gpu_max_FP_INT, = 0 +gpu_max_IC_H, = 0 +gpu_max_IC_M, = 0 +gpu_max_DC_RH, = 0 +gpu_max_DC_RM, = 0 +gpu_max_DC_WH, = 0 +gpu_max_DC_WM, = 0 +gpu_max_TC_H, = 0 +gpu_max_TC_M, = 0 +gpu_max_CC_H, = 0 +gpu_max_CC_M, = 0 +gpu_max_SHRD_ACC, = 0 +gpu_max_REG_RD, = 0 +gpu_max_REG_WR, = 0 +gpu_max_NON_REG_OPs, = 0 +gpu_max_SP_ACC, = 0 +gpu_max_SFU_ACC, = 0 +gpu_max_FPU_ACC, = 0 +gpu_max_MEM_RD, = 0 +gpu_max_MEM_WR, = 0 +gpu_max_MEM_PRE, = 0 +gpu_max_L2_RH, = 0 +gpu_max_L2_RM, = 0 +gpu_max_L2_WH, = 0 +gpu_max_L2_WM, = 0 +gpu_max_NOC_A, = 0 +gpu_max_PIPE_A, = 0 +gpu_max_IDLE_CORE_N, = 0 +gpu_max_CONST_DYNAMICN = 0 + +Kernel Minimum Power Data: +kernel_min_power = 0 +gpu_min_IBP, = 0 +gpu_min_ICP, = 0 +gpu_min_DCP, = 0 +gpu_min_TCP, = 0 +gpu_min_CCP, = 0 +gpu_min_SHRDP, = 0 +gpu_min_RFP, = 0 +gpu_min_SPP, = 0 +gpu_min_SFUP, = 0 +gpu_min_FPUP, = 0 +gpu_min_SCHEDP, = 0 +gpu_min_L2CP, = 0 +gpu_min_MCP, = 0 +gpu_min_NOCP, = 0 +gpu_min_DRAMP, = 0 +gpu_min_PIPEP, = 0 +gpu_min_IDLE_COREP, = 0 +gpu_min_CONST_DYNAMICP = 0 +gpu_min_TOT_INST, = 0 +gpu_min_FP_INT, = 0 +gpu_min_IC_H, = 0 +gpu_min_IC_M, = 0 +gpu_min_DC_RH, = 0 +gpu_min_DC_RM, = 0 +gpu_min_DC_WH, = 0 +gpu_min_DC_WM, = 0 +gpu_min_TC_H, = 0 +gpu_min_TC_M, = 0 +gpu_min_CC_H, = 0 +gpu_min_CC_M, = 0 +gpu_min_SHRD_ACC, = 0 +gpu_min_REG_RD, = 0 +gpu_min_REG_WR, = 0 +gpu_min_NON_REG_OPs, = 0 +gpu_min_SP_ACC, = 0 +gpu_min_SFU_ACC, = 0 +gpu_min_FPU_ACC, = 0 +gpu_min_MEM_RD, = 0 +gpu_min_MEM_WR, = 0 +gpu_min_MEM_PRE, = 0 +gpu_min_L2_RH, = 0 +gpu_min_L2_RM, = 0 +gpu_min_L2_WH, = 0 +gpu_min_L2_WM, = 0 +gpu_min_NOC_A, = 0 +gpu_min_PIPE_A, = 0 +gpu_min_IDLE_CORE_N, = 0 +gpu_min_CONST_DYNAMICN = 0 + +Accumulative Power Statistics Over Previous Kernels: +gpu_tot_avg_power = -nan +gpu_tot_max_power = 0 +gpu_tot_min_power = 0 + + +kernel_name = +kernel_launch_uid = + +Kernel Average Power Data: +kernel_avg_power = 0 +gpu_avg_IBP, = -nan +gpu_avg_ICP, = -nan +gpu_avg_DCP, = -nan +gpu_avg_TCP, = -nan +gpu_avg_CCP, = -nan +gpu_avg_SHRDP, = -nan +gpu_avg_RFP, = -nan +gpu_avg_SPP, = -nan +gpu_avg_SFUP, = -nan +gpu_avg_FPUP, = -nan +gpu_avg_SCHEDP, = -nan +gpu_avg_L2CP, = -nan +gpu_avg_MCP, = -nan +gpu_avg_NOCP, = -nan +gpu_avg_DRAMP, = -nan +gpu_avg_PIPEP, = -nan +gpu_avg_IDLE_COREP, = -nan +gpu_avg_CONST_DYNAMICP = -nan +gpu_avg_TOT_INST, = -nan +gpu_avg_FP_INT, = -nan +gpu_avg_IC_H, = -nan +gpu_avg_IC_M, = -nan +gpu_avg_DC_RH, = -nan +gpu_avg_DC_RM, = -nan +gpu_avg_DC_WH, = -nan +gpu_avg_DC_WM, = -nan +gpu_avg_TC_H, = -nan +gpu_avg_TC_M, = -nan +gpu_avg_CC_H, = -nan +gpu_avg_CC_M, = -nan +gpu_avg_SHRD_ACC, = -nan +gpu_avg_REG_RD, = -nan +gpu_avg_REG_WR, = -nan +gpu_avg_NON_REG_OPs, = -nan +gpu_avg_SP_ACC, = -nan +gpu_avg_SFU_ACC, = -nan +gpu_avg_FPU_ACC, = -nan +gpu_avg_MEM_RD, = -nan +gpu_avg_MEM_WR, = -nan +gpu_avg_MEM_PRE, = -nan +gpu_avg_L2_RH, = -nan +gpu_avg_L2_RM, = -nan +gpu_avg_L2_WH, = -nan +gpu_avg_L2_WM, = -nan +gpu_avg_NOC_A, = -nan +gpu_avg_PIPE_A, = -nan +gpu_avg_IDLE_CORE_N, = -nan +gpu_avg_CONST_DYNAMICN = -nan + +Kernel Maximum Power Data: +kernel_max_power = 0 +gpu_max_IBP, = 0 +gpu_max_ICP, = 0 +gpu_max_DCP, = 0 +gpu_max_TCP, = 0 +gpu_max_CCP, = 0 +gpu_max_SHRDP, = 0 +gpu_max_RFP, = 0 +gpu_max_SPP, = 0 +gpu_max_SFUP, = 0 +gpu_max_FPUP, = 0 +gpu_max_SCHEDP, = 0 +gpu_max_L2CP, = 0 +gpu_max_MCP, = 0 +gpu_max_NOCP, = 0 +gpu_max_DRAMP, = 0 +gpu_max_PIPEP, = 0 +gpu_max_IDLE_COREP, = 0 +gpu_max_CONST_DYNAMICP = 0 +gpu_max_TOT_INST, = 0 +gpu_max_FP_INT, = 0 +gpu_max_IC_H, = 0 +gpu_max_IC_M, = 0 +gpu_max_DC_RH, = 0 +gpu_max_DC_RM, = 0 +gpu_max_DC_WH, = 0 +gpu_max_DC_WM, = 0 +gpu_max_TC_H, = 0 +gpu_max_TC_M, = 0 +gpu_max_CC_H, = 0 +gpu_max_CC_M, = 0 +gpu_max_SHRD_ACC, = 0 +gpu_max_REG_RD, = 0 +gpu_max_REG_WR, = 0 +gpu_max_NON_REG_OPs, = 0 +gpu_max_SP_ACC, = 0 +gpu_max_SFU_ACC, = 0 +gpu_max_FPU_ACC, = 0 +gpu_max_MEM_RD, = 0 +gpu_max_MEM_WR, = 0 +gpu_max_MEM_PRE, = 0 +gpu_max_L2_RH, = 0 +gpu_max_L2_RM, = 0 +gpu_max_L2_WH, = 0 +gpu_max_L2_WM, = 0 +gpu_max_NOC_A, = 0 +gpu_max_PIPE_A, = 0 +gpu_max_IDLE_CORE_N, = 0 +gpu_max_CONST_DYNAMICN = 0 + +Kernel Minimum Power Data: +kernel_min_power = 0 +gpu_min_IBP, = 0 +gpu_min_ICP, = 0 +gpu_min_DCP, = 0 +gpu_min_TCP, = 0 +gpu_min_CCP, = 0 +gpu_min_SHRDP, = 0 +gpu_min_RFP, = 0 +gpu_min_SPP, = 0 +gpu_min_SFUP, = 0 +gpu_min_FPUP, = 0 +gpu_min_SCHEDP, = 0 +gpu_min_L2CP, = 0 +gpu_min_MCP, = 0 +gpu_min_NOCP, = 0 +gpu_min_DRAMP, = 0 +gpu_min_PIPEP, = 0 +gpu_min_IDLE_COREP, = 0 +gpu_min_CONST_DYNAMICP = 0 +gpu_min_TOT_INST, = 0 +gpu_min_FP_INT, = 0 +gpu_min_IC_H, = 0 +gpu_min_IC_M, = 0 +gpu_min_DC_RH, = 0 +gpu_min_DC_RM, = 0 +gpu_min_DC_WH, = 0 +gpu_min_DC_WM, = 0 +gpu_min_TC_H, = 0 +gpu_min_TC_M, = 0 +gpu_min_CC_H, = 0 +gpu_min_CC_M, = 0 +gpu_min_SHRD_ACC, = 0 +gpu_min_REG_RD, = 0 +gpu_min_REG_WR, = 0 +gpu_min_NON_REG_OPs, = 0 +gpu_min_SP_ACC, = 0 +gpu_min_SFU_ACC, = 0 +gpu_min_FPU_ACC, = 0 +gpu_min_MEM_RD, = 0 +gpu_min_MEM_WR, = 0 +gpu_min_MEM_PRE, = 0 +gpu_min_L2_RH, = 0 +gpu_min_L2_RM, = 0 +gpu_min_L2_WH, = 0 +gpu_min_L2_WM, = 0 +gpu_min_NOC_A, = 0 +gpu_min_PIPE_A, = 0 +gpu_min_IDLE_CORE_N, = 0 +gpu_min_CONST_DYNAMICN = 0 + +Accumulative Power Statistics Over Previous Kernels: +gpu_tot_avg_power = -nan +gpu_tot_max_power = 0 +gpu_tot_min_power = 0 + + diff --git a/cuda-kernels/gpuwattch_gtx1080Ti.xml b/cuda-kernels/gpuwattch_gtx1080Ti.xml new file mode 100755 index 0000000..02619ff --- /dev/null +++ b/cuda-kernels/gpuwattch_gtx1080Ti.xml @@ -0,0 +1,538 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cuda-kernels/tensor_core b/cuda-kernels/tensor_core new file mode 100755 index 0000000..b25f3d9 Binary files /dev/null and b/cuda-kernels/tensor_core differ diff --git a/cuda-kernels/tensor_core.cu b/cuda-kernels/tensor_core.cu new file mode 100644 index 0000000..483a42b --- /dev/null +++ b/cuda-kernels/tensor_core.cu @@ -0,0 +1,250 @@ +/* Copyright (c) 1993-2017, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +// Define some error checking macros. +#define cudaErrCheck(stat) { cudaErrCheck_((stat), __FILE__, __LINE__); } +void cudaErrCheck_(cudaError_t stat, const char *file, int line) { + if (stat != cudaSuccess) { + fprintf(stderr, "CUDA Error: %s %s %d\n", cudaGetErrorString(stat), file, line); + } +} + + + + +#include +using namespace nvcuda; + +// Must be multiples of 16 for wmma code to work +#define MATRIX_M (16) +#define MATRIX_N (16) +#define MATRIX_K (16) + + + +// The only dimensions currently supported by WMMA +const int WMMA_M = 16; +const int WMMA_N = 16; +const int WMMA_K = 16; + + +// Performs an MxNxK GEMM (C=alpha*A*B + beta*C) assuming: +// 1) Matrices are packed in memory. +// 2) M, N and K are multiples of 16. +// 3) Neither A nor B are transposed. +// Note: This is NOT a high performance example but is for demonstration purposes only +// For a high performance code please use the GEMM provided in cuBLAS. +__global__ void wmma_example(half *a, half *b, float *c, int M, int N, int K, float alpha, float beta) { + unsigned int start_time=0,end_time=0; + // Leading dimensions. Packed with no transpositions. + start_time=clock(); + int lda = M; + int ldb = K; + int ldc = M; + + // Tile using a 2D grid/ + int warpM = (blockIdx.x * blockDim.x + threadIdx.x) / warpSize; + int warpN = (blockIdx.y * blockDim.y + threadIdx.y); + + // Declare the fragments + wmma::fragment a_frag; + wmma::fragment b_frag; + wmma::fragment acc_frag; + wmma::fragment c_frag; + + wmma::fill_fragment(c_frag, 0.0f); + + int i=0; + int aRow = warpM * WMMA_M; + int bCol = warpN * WMMA_N; + int aCol = i; + int bRow = i; + + + // Bounds checking + if (aRow < M && aCol < K && bRow < K && bCol < N) { + wmma::load_matrix_sync(a_frag, a+aRow+aCol*lda, lda); + wmma::load_matrix_sync(b_frag, b+bRow*ldb+bCol, ldb); + wmma::mma_sync(c_frag, a_frag, b_frag, c_frag); + //wmma::mma_sync(acc_frag, a_frag, b_frag, acc_frag); + } + int cRow = warpM * WMMA_M; + int cCol = warpN * WMMA_N; + wmma::store_matrix_sync(c + cRow + cCol * ldc, c_frag, ldc, wmma::mem_col_major); + end_time=clock(); + printf("clock=%d",end_time-start_time); +} + +__global__ void convertFp32ToFp16 (half *out, float *in, int n) { + int idx = blockDim.x * blockIdx.x + threadIdx.x; + if (idx < n) { + out[idx] = in[idx]; + } +} + +int main(int argc, char* argv[]) { + float *a_fp32; + float *b_fp32; + half *a_fp16; + half *b_fp16; + + float *c; + float *c_cublas; + float *c_wmma; + + float *c_host_cublas; + float *c_host_wmma; + float *a_host_wmma; + float *b_host_wmma; + float *c_init_host_wmma; + + + cudaEvent_t startWMMA; + cudaEvent_t stopWMMA; + + + cudaErrCheck(cudaEventCreate(&startWMMA)); + cudaErrCheck(cudaEventCreate(&stopWMMA)); + + + + + // Use tensor cores + + + cudaErrCheck(cudaMalloc((void**)&a_fp32, MATRIX_M * MATRIX_K * sizeof(float))); + cudaErrCheck(cudaMalloc((void**)&b_fp32, MATRIX_K * MATRIX_N * sizeof(float))); + cudaErrCheck(cudaMalloc((void**)&a_fp16, MATRIX_M * MATRIX_K * sizeof(half))); + cudaErrCheck(cudaMalloc((void**)&b_fp16, MATRIX_K * MATRIX_N * sizeof(half))); + + cudaErrCheck(cudaMalloc((void**)&c, MATRIX_M * MATRIX_N * sizeof(float))); + cudaErrCheck(cudaMalloc((void**)&c_wmma, MATRIX_M * MATRIX_N * sizeof(float))); + + c_host_wmma = (float*)malloc(MATRIX_M * MATRIX_N * sizeof(float)); + c_init_host_wmma = (float*)malloc(MATRIX_M * MATRIX_N * sizeof(float)); + a_host_wmma = (float*)malloc(MATRIX_M * MATRIX_K * sizeof(float)); + b_host_wmma = (float*)malloc(MATRIX_K * MATRIX_N * sizeof(float)); + + + +// printf("a_fp32\n"); + for(int m=0;m>> (a_fp16, a_fp32, MATRIX_M * MATRIX_K); + convertFp32ToFp16 <<< (MATRIX_K * MATRIX_N + 255) / 256, 256 >>> (b_fp16, b_fp32, MATRIX_K * MATRIX_N); + + for(int m=0;m>> (a_fp16, b_fp16, c_wmma, MATRIX_M, MATRIX_N, MATRIX_K, alpha, beta); + // wmma_example <<< gridDim, blockDim >>> (a_fp16, b_fp16, c_wmma, MATRIX_M, MATRIX_N, MATRIX_K, alpha, beta); + cudaErrCheck(cudaEventRecord(stopWMMA)); + + + + + // Error checking + printf("\nChecking results...\n"); + cudaErrCheck(cudaMemcpy(c_host_wmma, c_wmma, MATRIX_M * MATRIX_N * sizeof(float), cudaMemcpyDeviceToHost)); + // printf("c_host\n"); + // for(int m=0;m{ +\.a\.sync TC; ptx_lval.int_value = LOAD_A; return WMMA_DIRECTIVE; +\.b\.sync TC; ptx_lval.int_value = LOAD_B; return WMMA_DIRECTIVE; +\.c\.sync TC; ptx_lval.int_value = LOAD_C; return WMMA_DIRECTIVE; +\.d\.sync TC; ptx_lval.int_value = STORE_D; return WMMA_DIRECTIVE; +\.sync TC;ptx_lval.int_value=MMA; return WMMA_DIRECTIVE; +\.row TC; ptx_lval.int_value = ROW; return LAYOUT; +\.col TC; ptx_lval.int_value = COL; return LAYOUT; +\.m16n16k16 TC; ptx_lval.int_value = M16N16K16; return CONFIGURATION; + \.align TC; return ALIGN_DIRECTIVE; \.branchtargets TC; return BRANCHTARGETS_DIRECTIVE; diff --git a/src/cuda-sim/ptx.y b/src/cuda-sim/ptx.y index 3360c55..737657c 100644 --- a/src/cuda-sim/ptx.y +++ b/src/cuda-sim/ptx.y @@ -37,6 +37,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %token STRING %token OPCODE +%token WMMA_DIRECTIVE +%token LAYOUT +%token CONFIGURATION %token ALIGN_DIRECTIVE %token BRANCHTARGETS_DIRECTIVE %token BYTE_DIRECTIVE @@ -428,6 +431,7 @@ option: type_spec | compare_spec | addressable_spec | rounding_mode + | wmma_spec | SYNC_OPTION { add_option(SYNC_OPTION); } | ARRIVE_OPTION { add_option(ARRIVE_OPTION); } | RED_OPTION { add_option(RED_OPTION); } @@ -483,6 +487,7 @@ atomic_operation_spec: ATOMIC_AND { add_option(ATOMIC_AND); } rounding_mode: floating_point_rounding_mode | integer_rounding_mode; + floating_point_rounding_mode: RN_OPTION { add_option(RN_OPTION); } | RZ_OPTION { add_option(RZ_OPTION); } | RM_OPTION { add_option(RM_OPTION); } @@ -515,6 +520,10 @@ compare_spec:EQ_OPTION { add_option(EQ_OPTION); } | NAN_OPTION { add_option(NAN_OPTION); } ; +wmma_spec: WMMA_DIRECTIVE LAYOUT CONFIGURATION{add_wmma_option($1);add_wmma_option($2);add_wmma_option($3);} + | WMMA_DIRECTIVE LAYOUT LAYOUT CONFIGURATION{add_wmma_option($1);add_wmma_option($2),add_wmma_option($3),add_wmma_option($4)} + ; + operand_list: operand | operand COMMA operand_list; @@ -543,6 +552,7 @@ operand: IDENTIFIER { add_scalar_operand( $1 ); } vector_operand: LEFT_BRACE IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { add_2vector_operand($2,$4); } | LEFT_BRACE IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { add_3vector_operand($2,$4,$6); } | LEFT_BRACE IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { add_4vector_operand($2,$4,$6,$8); } + | LEFT_BRACE IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { add_8vector_operand($2,$4,$6,$8,$10,$12,$14,$16); } | LEFT_BRACE IDENTIFIER RIGHT_BRACE { add_1vector_operand($2); } ; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index 8ebdcf8..9a4d8d3 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -995,7 +995,7 @@ static std::list check_operands( int opcode, const std::list &operands ) { static int g_warn_literal_operands_two_type_inst; - if( (opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || (opcode == TEX_OP) ) { + if( (opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || (opcode == TEX_OP) || (opcode==MMA_OP)) { // just make sure these do not have have const operands... if( !g_warn_literal_operands_two_type_inst ) { std::list::const_iterator o; @@ -1043,6 +1043,7 @@ ptx_instruction::ptx_instruction( int opcode, const std::list &operands, const operand_info &return_var, const std::list &options, + const std::list &wmma_options, const std::list &scalar_type, memory_space_t space_spec, const char *file, @@ -1061,6 +1062,7 @@ ptx_instruction::ptx_instruction( int opcode, m_operands.insert(m_operands.begin(), checked_operands.begin(), checked_operands.end() ); m_return_var = return_var; m_options = options; + m_wmma_options = wmma_options; m_wide = false; m_hi = false; m_lo = false; @@ -1078,7 +1080,7 @@ ptx_instruction::ptx_instruction( int opcode, m_atomic_spec = 0; m_membar_level = 0; m_inst_size = 8; // bytes - + int rr=0; std::list::const_iterator i; unsigned n=1; for ( i=options.begin(); i!= options.end(); i++, n++ ) { diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 0601b97..ff24a66 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -582,6 +582,34 @@ public: m_is_return_var = false; m_immediate_address=false; } + operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4 ,const symbol *s5,const symbol *s6,const symbol *s7, const symbol *s8) + { + init(); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = true; + m_type = vector_t; + m_value.m_vector_symbolic = new const symbol*[8]; + m_value.m_vector_symbolic[0] = s1; + m_value.m_vector_symbolic[1] = s2; + m_value.m_vector_symbolic[2] = s3; + m_value.m_vector_symbolic[3] = s4; + m_value.m_vector_symbolic[4] = s5; + m_value.m_vector_symbolic[5] = s6; + m_value.m_vector_symbolic[6] = s7; + m_value.m_vector_symbolic[7] = s8; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + void init() { m_uid=(unsigned)-1; @@ -866,6 +894,7 @@ public: const std::list &operands, const operand_info &return_var, const std::list &options, + const std::list &wmma_options, const std::list &scalar_type, memory_space_t space_spec, const char *file, @@ -1087,6 +1116,7 @@ private: operand_info m_return_var; std::list m_options; + std::list m_wmma_options; bool m_wide; bool m_hi; bool m_lo; @@ -1096,6 +1126,9 @@ private: bool m_uni; //if branch instruction, this evaluates to true for uniform branches (ie jumps) bool m_to_option; unsigned m_cache_option; + unsigned m_wmma_type; + unsigned m_wmma_layout[2]; + unsigned m_wmma_configuration; unsigned m_rounding_mode; unsigned m_compare_op; unsigned m_saturation_mode; diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index 7fc54e9..6757091 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -72,6 +72,7 @@ symbol *g_label; int g_opcode = -1; std::list g_operands; std::list g_options; +std::list g_wmma_options; std::list g_scalar_type; #define PTX_PARSE_DPRINTF(...) \ @@ -162,6 +163,7 @@ void init_instruction_state() g_label = NULL; g_opcode = -1; g_options.clear(); + g_wmma_options.clear(); g_return_var = operand_info(); init_directive_state(); } @@ -300,6 +302,7 @@ void add_instruction() g_operands, g_return_var, g_options, + g_wmma_options, g_scalar_type, g_space_spec, g_filename, @@ -629,7 +632,7 @@ void add_scalar_type_spec( int type_spec ) g_scalar_type.push_back( type_spec ); if ( g_scalar_type.size() > 1 ) { parse_assert( (g_opcode == -1) || (g_opcode == CVT_OP) || (g_opcode == SET_OP) || (g_opcode == SLCT_OP) - || (g_opcode == TEX_OP), + || (g_opcode == TEX_OP)|| (g_opcode==MMA_OP), "only cvt, set, slct, and tex can have more than one type specifier."); } g_scalar_type_spec = type_spec; @@ -669,7 +672,11 @@ void add_option( int option ) PTX_PARSE_DPRINTF("add_option"); g_options.push_back( option ); } - +void add_wmma_option( int option ) +{ + PTX_PARSE_DPRINTF("add_option"); + g_wmma_options.push_back( option ); +} void add_double_operand( const char *d1, const char *d2 ) { //operands that access two variables. @@ -725,6 +732,28 @@ void add_4vector_operand( const char *d1, const char *d2, const char *d3, const if ( s4 == null_op ) s4 = NULL; g_operands.push_back( operand_info(s1,s2,s3,s4) ); } +void add_8vector_operand( const char *d1, const char *d2, const char *d3, const char *d4,const char *d5,const char *d6,const char *d7,const char *d8 ) +{ + PTX_PARSE_DPRINTF("add_8vector_operand"); + const symbol *s1 = g_current_symbol_table->lookup(d1); + const symbol *s2 = g_current_symbol_table->lookup(d2); + const symbol *s3 = g_current_symbol_table->lookup(d3); + const symbol *s4 = g_current_symbol_table->lookup(d4); + const symbol *s5 = g_current_symbol_table->lookup(d5); + const symbol *s6 = g_current_symbol_table->lookup(d6); + const symbol *s7 = g_current_symbol_table->lookup(d7); + const symbol *s8 = g_current_symbol_table->lookup(d8); + parse_assert( s1 != NULL && s2 != NULL && s3 != NULL && s4 != NULL && s5 !=NULL && s6 !=NULL && s7 !=NULL && s8 !=NULL, "v4 component(s) missing declarations."); + const symbol *null_op = g_current_symbol_table->lookup("_"); + if ( s2 == null_op ) s2 = NULL; + if ( s3 == null_op ) s3 = NULL; + if ( s4 == null_op ) s4 = NULL; + if ( s5 == null_op ) s5 = NULL; + if ( s6 == null_op ) s6 = NULL; + if ( s7 == null_op ) s7 = NULL; + if ( s8 == null_op ) s8 = NULL; + g_operands.push_back( operand_info(s1,s2,s3,s4,s5,s6,s7,s8) ); +} void add_builtin_operand( int builtin, int dim_modifier ) { diff --git a/src/cuda-sim/ptx_parser.h b/src/cuda-sim/ptx_parser.h index 32f3903..8094b43 100644 --- a/src/cuda-sim/ptx_parser.h +++ b/src/cuda-sim/ptx_parser.h @@ -57,7 +57,9 @@ void add_1vector_operand( const char *d1 ); void add_2vector_operand( const char *d1, const char *d2 ); void add_3vector_operand( const char *d1, const char *d2, const char *d3 ); void add_4vector_operand( const char *d1, const char *d2, const char *d3, const char *d4 ); +void add_8vector_operand( const char *d1, const char *d2, const char *d3, const char *d4 ,const char *d5,const char *d6,const char *d7,const char *d8); void add_option(int option ); +void add_wmma_option(int option ); void add_builtin_operand( int builtin, int dim_modifier ); void add_memory_operand( ); void add_literal_int( int value ); -- cgit v1.3 From 5b1ba75a3d5d02fbc12b5218abaaae4fcf2b5c2d Mon Sep 17 00:00:00 2001 From: aamir Date: Wed, 30 May 2018 17:34:14 -0700 Subject: changes for vector operands --- src/abstract_hardware_model.h | 2 +- src/cuda-sim/cuda-sim.cc | 24 +++++++++---- src/cuda-sim/instructions.cc | 80 +++++++++++++++++++++++++++++++++---------- src/cuda-sim/ptx_ir.h | 10 +++++- 4 files changed, 88 insertions(+), 28 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index 9dc58d4..e00c941 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -826,7 +826,7 @@ public: address_type reconvergence_pc; // -1 => not a branch, -2 => use function return address unsigned out[8]; - unsigned in[8]; + unsigned in[24]; unsigned char is_vectorin; unsigned char is_vectorout; int pred; // predicate register number diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 006738a..62077e6 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -852,8 +852,10 @@ void ptx_instruction::pre_decode() { pc = m_PC; isize = m_inst_size; - for( unsigned i=0; i<4; i++) { + for(unsigned i=0; i<8; i++) { out[i] = 0; + } + for(unsigned i=0; i<24; i++) { in[i] = 0; } is_vectorin = 0; @@ -922,6 +924,10 @@ void ptx_instruction::pre_decode() if( num_elem >= 2 ) out[1] = o.reg2_num(); if( num_elem >= 3 ) out[2] = o.reg3_num(); if( num_elem >= 4 ) out[3] = o.reg4_num(); + if( num_elem >= 5 ) out[4] = o.reg5_num(); + if( num_elem >= 6 ) out[5] = o.reg6_num(); + if( num_elem >= 7 ) out[6] = o.reg7_num(); + if( num_elem >= 8 ) out[7] = o.reg8_num(); for (int i = 0; i < num_elem; i++) arch_reg.dst[i] = o.arch_reg_num(i); } @@ -940,13 +946,17 @@ void ptx_instruction::pre_decode() //assert(m == 0); //only support 1 vector operand (for textures) right now is_vectorout = 1; unsigned num_elem = o.get_vect_nelem(); - if( num_elem >= 1 ) in[0] = o.reg1_num(); - if( num_elem >= 2 ) in[1] = o.reg2_num(); - if( num_elem >= 3 ) in[2] = o.reg3_num(); - if( num_elem >= 4 ) in[3] = o.reg4_num(); + if( num_elem >= 1 ) in[m+0] = o.reg1_num(); + if( num_elem >= 2 ) in[m+1] = o.reg2_num(); + if( num_elem >= 3 ) in[m+2] = o.reg3_num(); + if( num_elem >= 4 ) in[m+3] = o.reg4_num(); + if( num_elem >= 5 ) in[m+4] = o.reg5_num(); + if( num_elem >= 6 ) in[m+5] = o.reg6_num(); + if( num_elem >= 7 ) in[m+6] = o.reg7_num(); + if( num_elem >= 8 ) in[m+7] = o.reg8_num(); for (int i = 0; i < num_elem; i++) - arch_reg.src[i] = o.arch_reg_num(i); - m+=4; + arch_reg.src[m+i] = o.arch_reg_num(i); + m+=num_elem; } } } diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 7407269..446cdbf 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -748,7 +748,7 @@ void addp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) case U64_TYPE: data.s64 = src1_data.s64 + src2_data.s64 + (src3_data.pred & 0x4); break; - case F16_TYPE: assert(0); break; + case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break; case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; default: assert(0); break; @@ -826,7 +826,7 @@ void add_impl( const ptx_instruction *pI, ptx_thread_info *thread ) case U64_TYPE: data.u64 = src1_data.u64 + src2_data.u64; break; - case F16_TYPE: assert(0); break; + case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break; case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; default: assert(0); break; @@ -1878,7 +1878,9 @@ ptx_reg_t f2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, } } else { switch ( to_width ) { - case 16: assert(0); break; + case 16: //assert(0); break; + y.f16 = x.f32; + break; case 32: assert(0); break; // handled by f2f case 64: y.f64 = x.f32; @@ -2140,7 +2142,7 @@ void ptx_round(ptx_reg_t& data, int rounding_mode, int type) case U32_TYPE: case U64_TYPE: printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: assert(0); break; + case F16_TYPE: data.f16=truncf(data.f16);break;//assert(0); break; case F32_TYPE: data.f32 = truncf(data.f32); break; @@ -2163,7 +2165,13 @@ void ptx_round(ptx_reg_t& data, int rounding_mode, int type) case U32_TYPE: case U64_TYPE: printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: assert(0); break; + case F16_TYPE:// assert(0); break; +#if CUDART_VERSION >= 3000 + data.f16 = nearbyintf(data.f16); +#else + data.f16 = cuda_math::__cuda_nearbyintf(data.f16); +#endif + break; case F32_TYPE: #if CUDART_VERSION >= 3000 data.f32 = nearbyintf(data.f32); @@ -2186,7 +2194,7 @@ void ptx_round(ptx_reg_t& data, int rounding_mode, int type) case U32_TYPE: case U64_TYPE: printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: assert(0); break; + case F16_TYPE: data.f16=floorf(data.f16);break;//assert(0); break; case F32_TYPE: data.f32 = floorf(data.f32); break; @@ -2205,7 +2213,7 @@ void ptx_round(ptx_reg_t& data, int rounding_mode, int type) case U32_TYPE: case U64_TYPE: printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: assert(0); break; + case F16_TYPE: data.f16 = ceilf(data.f16); break; //assert(0); break; case F32_TYPE: data.f32 = ceilf(data.f32); break; case F64_TYPE: case FF64_TYPE: data.f64 = ceil(data.f64); break; default: assert(0); break; @@ -2246,7 +2254,10 @@ void ptx_saturate(ptx_reg_t& data, int saturation_mode, int type) case U32_TYPE: case U64_TYPE: printf("Trying to clamp an integer to 1??\n"); assert(0); break; - case F16_TYPE: assert(0); break; + case F16_TYPE: //assert(0); break; + if (data.f16 > 1.0f) data.f16 = 1.0f; //negative + if (data.f16 < 0.0f) data.f16 = 0.0f; //positive + break; case F32_TYPE: if (data.f32 > 1.0f) data.f32 = 1.0f; //negative if (data.f32 < 0.0f) data.f32 = 0.0f; //positive @@ -2270,8 +2281,8 @@ void cvt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) unsigned rounding_mode = pI->rounding_mode(); unsigned saturation_mode = pI->saturation_mode(); - if ( to_type == F16_TYPE || from_type == F16_TYPE ) - abort(); +// if ( to_type == F16_TYPE || from_type == F16_TYPE ) +// abort(); int to_sign, from_sign; size_t from_width, to_width; @@ -2406,7 +2417,7 @@ void div_impl( const ptx_instruction *pI, ptx_thread_info *thread ) data.u32 = src1_data.u32 / src2_data.u32; break; case B64_TYPE: data.u64 = src1_data.u64 / src2_data.u64; break; - case F16_TYPE: assert(0); break; + case F16_TYPE: data.f16 = src1_data.f16 / src2_data.f16; break;//assert(0); break; case F32_TYPE: data.f32 = src1_data.f32 / src2_data.f32; break; case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 / src2_data.f64; break; default: assert(0); break; @@ -2744,9 +2755,24 @@ void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry if ( pI->is_lo() ) d.u64 = t.u64 + c.u64 + carry_bit.pred; else assert(0); break; - case F16_TYPE: - assert(0); - break; + case F16_TYPE:{ + // assert(0); + // break; + assert( use_carry == false); + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + d.f16 = a.f16 * b.f16 + c.f16; + if ( pI->saturation_mode() ) { + if ( d.f16 < 0 ) d.f16 = 0; + else if ( d.f16 > 1.0f ) d.f16 = 1.0f; + } + fesetround( orig_rm ); + break; + } case F32_TYPE: { assert( use_carry == false); int orig_rm = fegetround(); @@ -3046,9 +3072,25 @@ void mul_impl( const ptx_instruction *pI, ptx_thread_info *thread ) if ( pI->is_lo() ) d.u64 = t.u64; else assert(0); break; - case F16_TYPE: - assert(0); - break; + case F16_TYPE:{ + //assert(0); + //break; + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + + d.f16 = a.f16 * b.f16; + + if ( pI->saturation_mode() ) { + if ( d.f16 < 0 ) d.f16 = 0; + else if ( d.f16 > 1.0f ) d.f16 = 1.0f; + } + fesetround( orig_rm ); + break; + } case F32_TYPE: { int orig_rm = fegetround(); switch ( rounding_mode ) { @@ -3111,7 +3153,7 @@ void neg_impl( const ptx_instruction *pI, ptx_thread_info *thread ) case U32_TYPE: case U64_TYPE: assert(0); break; - case F16_TYPE: assert(0); break; + case F16_TYPE: data.f16 =0.0f - src1_data.f16; break;//assert(0); break; case F32_TYPE: data.f32 = 0.0f - src1_data.f32; break; case F64_TYPE: case FF64_TYPE: data.f64 = 0.0f - src1_data.f64; break; default: assert(0); break; @@ -4165,7 +4207,7 @@ void sub_impl( const ptx_instruction *pI, ptx_thread_info *thread ) case B64_TYPE: case U64_TYPE: data.u64 = src1_data.u64 - src2_data.u64; break; - case F16_TYPE: assert(0); break; + case F16_TYPE: data.f16 = src1_data.f16 - src2_data.f16; break;//assert(0); break; case F32_TYPE: data.f32 = src1_data.f32 - src2_data.f32; break; case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 - src2_data.f64; break; default: assert(0); break; diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index ff24a66..833f175 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -656,7 +656,11 @@ public: if( !m_value.m_vector_symbolic[1] ) return 1; if( !m_value.m_vector_symbolic[2] ) return 2; if( !m_value.m_vector_symbolic[3] ) return 3; - return 4; + if( !m_value.m_vector_symbolic[4] ) return 4; + if( !m_value.m_vector_symbolic[5] ) return 5; + if( !m_value.m_vector_symbolic[6] ) return 6; + if( !m_value.m_vector_symbolic[7] ) return 7; + return 8; } const symbol* vec_symbol(int idx) const @@ -718,6 +722,10 @@ public: int reg2_num() const { return m_value.m_vector_symbolic[1]->reg_num();} int reg3_num() const { return m_value.m_vector_symbolic[2]?m_value.m_vector_symbolic[2]->reg_num():0; } int reg4_num() const { return m_value.m_vector_symbolic[3]?m_value.m_vector_symbolic[3]->reg_num():0; } + int reg5_num() const { return m_value.m_vector_symbolic[4]?m_value.m_vector_symbolic[4]->reg_num():0; } + int reg6_num() const { return m_value.m_vector_symbolic[5]?m_value.m_vector_symbolic[5]->reg_num():0; } + int reg7_num() const { return m_value.m_vector_symbolic[6]?m_value.m_vector_symbolic[6]->reg_num():0; } + int reg8_num() const { return m_value.m_vector_symbolic[7]?m_value.m_vector_symbolic[7]->reg_num():0; } int arch_reg_num() const { return m_value.m_symbolic->arch_reg_num(); } int arch_reg_num(unsigned n) const { return (m_value.m_vector_symbolic[n])? m_value.m_vector_symbolic[n]->arch_reg_num() : -1; } bool is_label() const { return m_type == label_t;} -- cgit v1.3 From fa0089a5d3a86ef348fae9a83a862f5219892bab Mon Sep 17 00:00:00 2001 From: aamir Date: Wed, 30 May 2018 23:07:44 -0700 Subject: adding code for wmma_ld_impl, error at decode space --- cuda-kernels/.tensor_core_ptx.swp | Bin 0 -> 16384 bytes cuda-kernels/Makefile | 3 +- cuda-kernels/tensor_core | Bin 48541 -> 2750968 bytes cuda-kernels/tensor_core_ptx | 171 +++++++++++ src/Makefile | 2 +- src/cuda-sim/.ptx.y.swp | Bin 0 -> 36864 bytes src/cuda-sim/Makefile | 2 +- src/cuda-sim/instructions.cc | 72 ++++- src/cuda-sim/ptx.y~ | 608 ++++++++++++++++++++++++++++++++++++++ src/cuda-sim/ptx_ir.h | 4 +- src/cuda-sim/ptx_sim.h | 9 + src/gpgpu-sim/Makefile | 2 +- src/intersim2/Makefile | 2 +- 13 files changed, 865 insertions(+), 10 deletions(-) create mode 100644 cuda-kernels/.tensor_core_ptx.swp create mode 100644 cuda-kernels/tensor_core_ptx create mode 100644 src/cuda-sim/.ptx.y.swp create mode 100644 src/cuda-sim/ptx.y~ (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/cuda-kernels/.tensor_core_ptx.swp b/cuda-kernels/.tensor_core_ptx.swp new file mode 100644 index 0000000..6d7bad4 Binary files /dev/null and b/cuda-kernels/.tensor_core_ptx.swp differ diff --git a/cuda-kernels/Makefile b/cuda-kernels/Makefile index 51a7760..673460f 100755 --- a/cuda-kernels/Makefile +++ b/cuda-kernels/Makefile @@ -1,5 +1,6 @@ all: tensor_core.cu - nvcc -arch=sm_70 -lcudart -g -o tensor_core tensor_core.cu + nvcc --gpu-architecture=compute_70 --gpu-code=compute_70 -lcudart -g -o tensor_core tensor_core.cu +# nvcc -arch=sm_70 -lcudart -g -o tensor_core tensor_core.cu .PHONY: clean: diff --git a/cuda-kernels/tensor_core b/cuda-kernels/tensor_core index b25f3d9..cb53851 100755 Binary files a/cuda-kernels/tensor_core and b/cuda-kernels/tensor_core differ diff --git a/cuda-kernels/tensor_core_ptx b/cuda-kernels/tensor_core_ptx new file mode 100644 index 0000000..36074cb --- /dev/null +++ b/cuda-kernels/tensor_core_ptx @@ -0,0 +1,171 @@ +// +// Generated by NVIDIA NVVM Compiler +// +// Compiler Build ID: CL-22781540 +// Cuda compilation tools, release 9.0, V9.0.176 +// Based on LLVM 3.4svn +// + +.version 6.0 +.target sm_70 +.address_size 64 + + // .globl _Z12wmma_exampleP6__halfS0_Pfiiiff +.extern .func (.param .b32 func_retval0) vprintf +( + .param .b64 vprintf_param_0, + .param .b64 vprintf_param_1 +) +; +.global .align 16 .b8 $str[9] = {99, 108, 111, 99, 107, 61, 37, 100, 0}; + +.visible .entry _Z12wmma_exampleP6__halfS0_Pfiiiff( + .param .u64 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_0, + .param .u64 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_1, + .param .u64 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_2, + .param .u32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_3, + .param .u32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_4, + .param .u32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_5, + .param .f32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_6, + .param .f32 _Z12wmma_exampleP6__halfS0_Pfiiiff_param_7 +) +{ + .local .align 8 .b8 __local_depot0[8]; + .reg .b64 %SP; + .reg .b64 %SPL; + .reg .pred %p<6>; + .reg .f32 %f<34>; + .reg .b32 %r<38>; + .reg .b64 %rd<18>; + + + mov.u64 %rd17, __local_depot0; + cvta.local.u64 %SP, %rd17; + ld.param.u64 %rd1, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_0]; + ld.param.u64 %rd2, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_1]; + ld.param.u64 %rd3, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_2]; + ld.param.u32 %r4, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_3]; + ld.param.u32 %r7, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_4]; + ld.param.u32 %r5, [_Z12wmma_exampleP6__halfS0_Pfiiiff_param_5]; + // inline asm + mov.u32 %r6, %clock; + // inline asm + mov.u32 %r8, %ntid.x; + mov.u32 %r9, %ctaid.x; + mov.u32 %r10, %tid.x; + mad.lo.s32 %r11, %r8, %r9, %r10; + mov.u32 %r12, WARP_SZ; + div.u32 %r13, %r11, %r12; + mov.u32 %r14, %ntid.y; + mov.u32 %r15, %ctaid.y; + mov.u32 %r16, %tid.y; + mad.lo.s32 %r17, %r14, %r15, %r16; + shl.b32 %r2, %r13, 4; + shl.b32 %r3, %r17, 4; + setp.lt.s32 %p1, %r2, %r4; + setp.gt.s32 %p2, %r5, 0; + and.pred %p3, %p1, %p2; + setp.lt.s32 %p4, %r3, %r7; + and.pred %p5, %p3, %p4; + mov.f32 %f26, 0f00000000; + mov.f32 %f27, %f26; + mov.f32 %f28, %f26; + mov.f32 %f29, %f26; + mov.f32 %f30, %f26; + mov.f32 %f31, %f26; + mov.f32 %f32, %f26; + mov.f32 %f33, %f26; + @!%p5 bra BB0_2; + bra.uni BB0_1; + +BB0_1: + mul.wide.s32 %rd4, %r2, 2; + add.s64 %rd5, %rd1, %rd4; + wmma.load.a.sync.row.m16n16k16.f16 {%r18, %r19, %r20, %r21, %r22, %r23, %r24, %r25}, [%rd5], %r4; + mul.wide.s32 %rd6, %r3, 2; + add.s64 %rd7, %rd2, %rd6; + wmma.load.b.sync.col.m16n16k16.f16 {%r26, %r27, %r28, %r29, %r30, %r31, %r32, %r33}, [%rd7], %r5; + mov.f32 %f25, 0f00000000; + wmma.mma.sync.row.col.m16n16k16.f32.f32 {%f33, %f32, %f31, %f30, %f29, %f28, %f27, %f26}, {%r18, %r19, %r20, %r21, %r22, %r23, %r24, %r25}, {%r26, %r27, %r28, %r29, %r30, %r31, %r32, %r33}, {%f25, %f25, %f25, %f25, %f25, %f25, %f25, %f25}; + +BB0_2: + add.u64 %rd8, %SP, 0; + cvta.to.local.u64 %rd9, %rd8; + mul.lo.s32 %r35, %r3, %r4; + cvt.s64.s32 %rd10, %r35; + cvt.s64.s32 %rd11, %r2; + add.s64 %rd12, %rd10, %rd11; + shl.b64 %rd13, %rd12, 2; + add.s64 %rd14, %rd3, %rd13; + wmma.store.d.sync.col.m16n16k16.f32 [%rd14], {%f33, %f32, %f31, %f30, %f29, %f28, %f27, %f26}, %r4; + // inline asm + mov.u32 %r34, %clock; + // inline asm + sub.s32 %r36, %r34, %r6; + st.local.u32 [%rd9], %r36; + mov.u64 %rd15, $str; + cvta.global.u64 %rd16, %rd15; + // Callseq Start 0 + { + .reg .b32 temp_param_reg; + // } + .param .b64 param0; + st.param.b64 [param0+0], %rd16; + .param .b64 param1; + st.param.b64 [param1+0], %rd8; + .param .b32 retval0; + call.uni (retval0), + vprintf, + ( + param0, + param1 + ); + ld.param.b32 %r37, [retval0+0]; + + //{ + }// Callseq End 0 + ret; +} + + // .globl _Z17convertFp32ToFp16P6__halfPfi +.visible .entry _Z17convertFp32ToFp16P6__halfPfi( + .param .u64 _Z17convertFp32ToFp16P6__halfPfi_param_0, + .param .u64 _Z17convertFp32ToFp16P6__halfPfi_param_1, + .param .u32 _Z17convertFp32ToFp16P6__halfPfi_param_2 +) +{ + .reg .pred %p<2>; + .reg .b16 %rs<2>; + .reg .f32 %f<2>; + .reg .b32 %r<6>; + .reg .b64 %rd<9>; + + + ld.param.u64 %rd1, [_Z17convertFp32ToFp16P6__halfPfi_param_0]; + ld.param.u64 %rd2, [_Z17convertFp32ToFp16P6__halfPfi_param_1]; + ld.param.u32 %r2, [_Z17convertFp32ToFp16P6__halfPfi_param_2]; + mov.u32 %r3, %ntid.x; + mov.u32 %r4, %ctaid.x; + mov.u32 %r5, %tid.x; + mad.lo.s32 %r1, %r4, %r3, %r5; + setp.ge.s32 %p1, %r1, %r2; + @%p1 bra BB1_2; + + cvta.to.global.u64 %rd3, %rd2; + mul.wide.s32 %rd4, %r1, 4; + add.s64 %rd5, %rd3, %rd4; + ld.global.f32 %f1, [%rd5]; + // inline asm + { cvt.rn.f16.f32 %rs1, %f1;} + + // inline asm + cvta.to.global.u64 %rd6, %rd1; + mul.wide.s32 %rd7, %r1, 2; + add.s64 %rd8, %rd6, %rd7; + st.global.u16 [%rd8], %rs1; + +BB1_2: + ret; +} + + diff --git a/src/Makefile b/src/Makefile index 6001669..09194f3 100644 --- a/src/Makefile +++ b/src/Makefile @@ -46,7 +46,7 @@ ifeq ($(TRACE),1) endif ifneq ($(DEBUG),1) - OPTFLAGS += -O3 + OPTFLAGS += -O0 else CXXFLAGS += endif diff --git a/src/cuda-sim/.ptx.y.swp b/src/cuda-sim/.ptx.y.swp new file mode 100644 index 0000000..c8a83b5 Binary files /dev/null and b/src/cuda-sim/.ptx.y.swp differ diff --git a/src/cuda-sim/Makefile b/src/cuda-sim/Makefile index 999dad7..a65e8e1 100644 --- a/src/cuda-sim/Makefile +++ b/src/cuda-sim/Makefile @@ -42,7 +42,7 @@ include ../../version_detection.mk OUTPUT_DIR=$(SIM_OBJ_FILES_DIR)/cuda-sim -OPT := -O3 -g3 -Wall -Wno-unused-function -Wno-sign-compare +OPT := -O0 -g3 -Wall -Wno-unused-function -Wno-sign-compare ifeq ($(DEBUG),1) OPT := -g3 -Wall -Wno-unused-function -Wno-sign-compare endif diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 446cdbf..16f33c6 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -643,6 +643,33 @@ void ptx_thread_info::set_vector_operand_values( const operand_info &dst, m_last_set_operand_value = data1; } +void ptx_thread_info::set_wmma_vector_operand_values( const operand_info &dst, + const ptx_reg_t &data1, + const ptx_reg_t &data2, + const ptx_reg_t &data3, + const ptx_reg_t &data4, + const ptx_reg_t &data5, + const ptx_reg_t &data6, + const ptx_reg_t &data7, + const ptx_reg_t &data8 ) +{ + unsigned num_elements = dst.get_vect_nelem(); + if (num_elements > 7) { + set_reg(dst.vec_symbol(0), data1); + set_reg(dst.vec_symbol(1), data2); + set_reg(dst.vec_symbol(2), data3); + set_reg(dst.vec_symbol(3), data4); + set_reg(dst.vec_symbol(4), data5); + set_reg(dst.vec_symbol(5), data6); + set_reg(dst.vec_symbol(6), data7); + set_reg(dst.vec_symbol(7), data8); + } + else{ + printf("error:set_wmma_vector_operands"); + } + + m_last_set_operand_value = data1; +} #define my_abs(a) (((a)<0)?(-a):(a)) @@ -1493,9 +1520,6 @@ unsigned trunc(unsigned num, unsigned precision) { } return num; } -void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) -{ -} void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) { } @@ -2595,6 +2619,48 @@ void ldu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { ld_exec(pI,thread); } +void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned type = pI->get_type(); + + int tid = inst.warp_id_func() * core->get_warp_size(); + int thrd; + ptx_thread_info *thread; + thread = core->get_thread_info()[tid]; + + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, type, thread, 1); + + ptx_reg_t data; + memory_space_t space = pI->get_space(); + + memory_space *mem = NULL; + addr_t addr = src1_data.u32; + + decode_space(space,thread,src1,mem,addr); + + size_t size; + int t; + data.u64=0; + type_info_key::type_decode(type,size,t); + ptx_reg_t data1, data2, data3, data4; + ptx_reg_t data5, data6, data7, data8; + mem->read(addr,size/8,&data1.s64); + mem->read(addr+size/8,size/8,&data2.s64); + mem->read(addr+2*size/8,size/8,&data3.s64); + mem->read(addr+3*size/8,size/8,&data4.s64); + mem->read(addr+4*size/8,size/8,&data5.s64); + mem->read(addr+5*size/8,size/8,&data6.s64); + mem->read(addr+6*size/8,size/8,&data7.s64); + mem->read(addr+7*size/8,size/8,&data8.s64); + thread->set_wmma_vector_operand_values(dst,data1,data2,data3,data4,data5,data6,data7,data8); + + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; +} void lg2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { diff --git a/src/cuda-sim/ptx.y~ b/src/cuda-sim/ptx.y~ new file mode 100644 index 0000000..0710ecd --- /dev/null +++ b/src/cuda-sim/ptx.y~ @@ -0,0 +1,608 @@ +/* +Copyright (c) 2009-2011, Tor M. Aamodt +The University of British Columbia +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. +Neither the name of The University of British Columbia nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +%union { + double double_value; + float float_value; + int int_value; + char * string_value; + void * ptr_value; +} + +%token STRING +%token OPCODE +%token WMMA_DIRECTIVE +%token LAYOUT +%token CONFIGURATION +%token ALIGN_DIRECTIVE +%token BRANCHTARGETS_DIRECTIVE +%token BYTE_DIRECTIVE +%token CALLPROTOTYPE_DIRECTIVE +%token CALLTARGETS_DIRECTIVE +%token CONST_DIRECTIVE +%token CONSTPTR_DIRECTIVE +%token PTR_DIRECTIVE +%token ENTRY_DIRECTIVE +%token EXTERN_DIRECTIVE +%token WEAK_DIRECTIVE +%token FILE_DIRECTIVE +%token FUNC_DIRECTIVE +%token GLOBAL_DIRECTIVE +%token LOCAL_DIRECTIVE +%token LOC_DIRECTIVE +%token MAXNCTAPERSM_DIRECTIVE +%token MAXNNREG_DIRECTIVE +%token MAXNTID_DIRECTIVE +%token MINNCTAPERSM_DIRECTIVE +%token PARAM_DIRECTIVE +%token PRAGMA_DIRECTIVE +%token REG_DIRECTIVE +%token REQNTID_DIRECTIVE +%token SECTION_DIRECTIVE +%token SHARED_DIRECTIVE +%token SREG_DIRECTIVE +%token SSTARR_DIRECTIVE +%token STRUCT_DIRECTIVE +%token SURF_DIRECTIVE +%token TARGET_DIRECTIVE +%token TEX_DIRECTIVE +%token UNION_DIRECTIVE +%token VERSION_DIRECTIVE +%token ADDRESS_SIZE_DIRECTIVE +%token VISIBLE_DIRECTIVE +%token WEAK_DIRECTIVE +%token IDENTIFIER +%token INT_OPERAND +%token FLOAT_OPERAND +%token DOUBLE_OPERAND +%token S8_TYPE +%token S16_TYPE +%token S32_TYPE +%token S64_TYPE +%token U8_TYPE +%token U16_TYPE +%token U32_TYPE +%token U64_TYPE +%token F16_TYPE +%token F32_TYPE +%token F64_TYPE +%token FF64_TYPE +%token B8_TYPE +%token B16_TYPE +%token B32_TYPE +%token B64_TYPE +%token BB64_TYPE +%token BB128_TYPE +%token PRED_TYPE +%token TEXREF_TYPE +%token SAMPLERREF_TYPE +%token SURFREF_TYPE +%token V2_TYPE +%token V3_TYPE +%token V4_TYPE +%token COMMA +%token PRED +%token HALF_OPTION +%token EXTP_OPTION +%token EQ_OPTION +%token NE_OPTION +%token LT_OPTION +%token LE_OPTION +%token GT_OPTION +%token GE_OPTION +%token LO_OPTION +%token LS_OPTION +%token HI_OPTION +%token HS_OPTION +%token EQU_OPTION +%token NEU_OPTION +%token LTU_OPTION +%token LEU_OPTION +%token GTU_OPTION +%token GEU_OPTION +%token NUM_OPTION +%token NAN_OPTION +%token CF_OPTION +%token SF_OPTION +%token NSF_OPTION +%token LEFT_SQUARE_BRACKET +%token RIGHT_SQUARE_BRACKET +%token WIDE_OPTION +%token SPECIAL_REGISTER +%token MINUS +%token PLUS +%token COLON +%token SEMI_COLON +%token EXCLAMATION +%token PIPE +%token RIGHT_BRACE +%token LEFT_BRACE +%token EQUALS +%token PERIOD +%token BACKSLASH +%token DIMENSION_MODIFIER +%token RN_OPTION +%token RZ_OPTION +%token RM_OPTION +%token RP_OPTION +%token RNI_OPTION +%token RZI_OPTION +%token RMI_OPTION +%token RPI_OPTION +%token UNI_OPTION +%token GEOM_MODIFIER_1D +%token GEOM_MODIFIER_2D +%token GEOM_MODIFIER_3D +%token SAT_OPTION +%token FTZ_OPTION +%token NEG_OPTION +%token SYNC_OPTION +%token RED_OPTION +%token ARRIVE_OPTION +%token ATOMIC_POPC +%token ATOMIC_AND +%token ATOMIC_OR +%token ATOMIC_XOR +%token ATOMIC_CAS +%token ATOMIC_EXCH +%token ATOMIC_ADD +%token ATOMIC_INC +%token ATOMIC_DEC +%token ATOMIC_MIN +%token ATOMIC_MAX +%token LEFT_ANGLE_BRACKET +%token RIGHT_ANGLE_BRACKET +%token LEFT_PAREN +%token RIGHT_PAREN +%token APPROX_OPTION +%token FULL_OPTION +%token ANY_OPTION +%token ALL_OPTION +%token BALLOT_OPTION +%token GLOBAL_OPTION +%token CTA_OPTION +%token SYS_OPTION +%token EXIT_OPTION +%token ABS_OPTION +%token TO_OPTION +%token CA_OPTION; +%token CG_OPTION; +%token CS_OPTION; +%token LU_OPTION; +%token CV_OPTION; +%token WB_OPTION; +%token WT_OPTION; +%token NC_OPTION; +%token UP_OPTION; +%token DOWN_OPTION; +%token BFLY_OPTION; +%token IDX_OPTION; + +%type function_decl_header +%type function_decl + +%{ + #include "ptx_parser.h" + #include + #include + #include + void syntax_not_implemented(); + extern int g_func_decl; + int ptx_lex(void); + int ptx_error(const char *); +%} + +%% + +input: /* empty */ + | input directive_statement + | input function_defn + | input function_decl + ; + +function_defn: function_decl { set_symtab($1); func_header(".skip"); } statement_block { end_function(); } + | function_decl { set_symtab($1); } block_spec_list { func_header(".skip"); } statement_block { end_function(); } + ; + +block_spec: MAXNTID_DIRECTIVE INT_OPERAND COMMA INT_OPERAND COMMA INT_OPERAND {func_header_info_int(".maxntid", $2); + func_header_info_int(",", $4); + func_header_info_int(",", $6); } + | MINNCTAPERSM_DIRECTIVE INT_OPERAND { func_header_info_int(".minnctapersm", $2); printf("GPGPU-Sim: Warning: .minnctapersm ignored. \n"); } + | MAXNCTAPERSM_DIRECTIVE INT_OPERAND { func_header_info_int(".maxnctapersm", $2); printf("GPGPU-Sim: Warning: .maxnctapersm ignored. \n"); } + ; + +block_spec_list: block_spec + | block_spec_list block_spec + ; + +function_decl: function_decl_header LEFT_PAREN { start_function($1); func_header_info("(");} param_entry RIGHT_PAREN {func_header_info(")");} function_ident_param { $$ = reset_symtab(); } + | function_decl_header { start_function($1); } function_ident_param { $$ = reset_symtab(); } + | function_decl_header { start_function($1); add_function_name(""); g_func_decl=0; $$ = reset_symtab(); } + ; + +function_ident_param: IDENTIFIER { add_function_name($1); } LEFT_PAREN {func_header_info("(");} param_list RIGHT_PAREN { g_func_decl=0; func_header_info(")"); } + | IDENTIFIER { add_function_name($1); g_func_decl=0; } + ; + +function_decl_header: ENTRY_DIRECTIVE { $$ = 1; g_func_decl=1; func_header(".entry"); } + | VISIBLE_DIRECTIVE ENTRY_DIRECTIVE { $$ = 1; g_func_decl=1; func_header(".entry"); } + | WEAK_DIRECTIVE ENTRY_DIRECTIVE { $$ = 1; g_func_decl=1; func_header(".entry"); } + | FUNC_DIRECTIVE { $$ = 0; g_func_decl=1; func_header(".func"); } + | VISIBLE_DIRECTIVE FUNC_DIRECTIVE { $$ = 0; g_func_decl=1; func_header(".func"); } + | WEAK_DIRECTIVE FUNC_DIRECTIVE { $$ = 0; g_func_decl=1; func_header(".func"); } + | EXTERN_DIRECTIVE FUNC_DIRECTIVE { $$ = 2; g_func_decl=1; func_header(".func"); } + | WEAK_DIRECTIVE FUNC_DIRECTIVE { $$ = 0; g_func_decl=1; func_header(".func"); } + ; + +param_list: /*empty*/ + | param_entry { add_directive(); } + | param_list COMMA {func_header_info(",");} param_entry { add_directive(); } + +param_entry: PARAM_DIRECTIVE { add_space_spec(param_space_unclassified,0); } variable_spec ptr_spec identifier_spec { add_function_arg(); } + | REG_DIRECTIVE { add_space_spec(reg_space,0); } variable_spec identifier_spec { add_function_arg(); } + +ptr_spec: /*empty*/ + | PTR_DIRECTIVE ptr_space_spec ptr_align_spec + | PTR_DIRECTIVE ptr_align_spec + +ptr_space_spec: GLOBAL_DIRECTIVE { add_ptr_spec(global_space); } + | LOCAL_DIRECTIVE { add_ptr_spec(local_space); } + | SHARED_DIRECTIVE { add_ptr_spec(shared_space); } + +ptr_align_spec: ALIGN_DIRECTIVE INT_OPERAND + +statement_block: LEFT_BRACE statement_list RIGHT_BRACE + +statement_list: directive_statement { add_directive(); } + | instruction_statement { add_instruction(); } + | statement_list directive_statement { add_directive(); } + | statement_list instruction_statement { add_instruction(); } + | statement_list {start_inst_group();} statement_block {end_inst_group();} + | {start_inst_group();} statement_block {end_inst_group();} + ; + +directive_statement: variable_declaration SEMI_COLON + | VERSION_DIRECTIVE DOUBLE_OPERAND { add_version_info($2, 0); } + | VERSION_DIRECTIVE DOUBLE_OPERAND PLUS { add_version_info($2,1); } + | ADDRESS_SIZE_DIRECTIVE INT_OPERAND {/*Do nothing*/} + | TARGET_DIRECTIVE IDENTIFIER COMMA IDENTIFIER { target_header2($2,$4); } + | TARGET_DIRECTIVE IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER { target_header3($2,$4,$6); } + | TARGET_DIRECTIVE IDENTIFIER { target_header($2); } + | FILE_DIRECTIVE INT_OPERAND STRING { add_file($2,$3); } + | FILE_DIRECTIVE INT_OPERAND STRING COMMA INT_OPERAND COMMA INT_OPERAND { add_file($2,$3); } + | LOC_DIRECTIVE INT_OPERAND INT_OPERAND INT_OPERAND + | PRAGMA_DIRECTIVE STRING SEMI_COLON { add_pragma($2); } + | function_decl SEMI_COLON {/*Do nothing*/} + ; + +variable_declaration: variable_spec identifier_list { add_variables(); } + | variable_spec identifier_spec EQUALS initializer_list { add_variables(); } + | variable_spec identifier_spec EQUALS literal_operand { add_variables(); } + | CONSTPTR_DIRECTIVE IDENTIFIER COMMA IDENTIFIER COMMA INT_OPERAND { add_constptr($2, $4, $6); } + ; + +variable_spec: var_spec_list { set_variable_type(); } + +identifier_list: identifier_spec + | identifier_list COMMA identifier_spec; + +identifier_spec: IDENTIFIER { add_identifier($1,0,NON_ARRAY_IDENTIFIER); func_header_info($1);} + | IDENTIFIER LEFT_ANGLE_BRACKET INT_OPERAND RIGHT_ANGLE_BRACKET { func_header_info($1); func_header_info_int("<", $3); func_header_info(">"); + int i,lbase,l; + char *id = NULL; + lbase = strlen($1); + for( i=0; i < $3; i++ ) { + l = lbase + (int)log10(i+1)+10; + id = (char*) malloc(l); + snprintf(id,l,"%s%u",$1,i); + add_identifier(id,0,NON_ARRAY_IDENTIFIER); + } + free($1); + } + | IDENTIFIER LEFT_SQUARE_BRACKET RIGHT_SQUARE_BRACKET { add_identifier($1,0,ARRAY_IDENTIFIER_NO_DIM); func_header_info($1); func_header_info("["); func_header_info("]");} + | IDENTIFIER LEFT_SQUARE_BRACKET INT_OPERAND RIGHT_SQUARE_BRACKET { add_identifier($1,$3,ARRAY_IDENTIFIER); func_header_info($1); func_header_info_int("[",$3); func_header_info("]");} + ; + +var_spec_list: var_spec + | var_spec_list var_spec; + +var_spec: space_spec + | type_spec + | align_spec + | EXTERN_DIRECTIVE { add_extern_spec(); } + | WEAK_DIRECTIVE + ; + +align_spec: ALIGN_DIRECTIVE INT_OPERAND { add_alignment_spec($2); } + +space_spec: REG_DIRECTIVE { add_space_spec(reg_space,0); } + | SREG_DIRECTIVE { add_space_spec(reg_space,0); } + | addressable_spec + ; + +addressable_spec: CONST_DIRECTIVE { add_space_spec(const_space,$1); } + | GLOBAL_DIRECTIVE { add_space_spec(global_space,0); } + | LOCAL_DIRECTIVE { add_space_spec(local_space,0); } + | PARAM_DIRECTIVE { add_space_spec(param_space_unclassified,0); } + | SHARED_DIRECTIVE { add_space_spec(shared_space,0); } + | SSTARR_DIRECTIVE { add_space_spec(sstarr_space,0); } + | SURF_DIRECTIVE { add_space_spec(surf_space,0); } + | TEX_DIRECTIVE { add_space_spec(tex_space,0); } + ; + +type_spec: scalar_type + | vector_spec scalar_type + ; + +vector_spec: V2_TYPE { add_option(V2_TYPE); func_header_info(".v2");} + | V3_TYPE { add_option(V3_TYPE); func_header_info(".v3");} + | V4_TYPE { add_option(V4_TYPE); func_header_info(".v4");} + ; + +scalar_type: S8_TYPE { add_scalar_type_spec( S8_TYPE ); } + | S16_TYPE { add_scalar_type_spec( S16_TYPE ); } + | S32_TYPE { add_scalar_type_spec( S32_TYPE ); } + | S64_TYPE { add_scalar_type_spec( S64_TYPE ); } + | U8_TYPE { add_scalar_type_spec( U8_TYPE ); } + | U16_TYPE { add_scalar_type_spec( U16_TYPE ); } + | U32_TYPE { add_scalar_type_spec( U32_TYPE ); } + | U64_TYPE { add_scalar_type_spec( U64_TYPE ); } + | F16_TYPE { add_scalar_type_spec( F16_TYPE ); } + | F32_TYPE { add_scalar_type_spec( F32_TYPE ); } + | F64_TYPE { add_scalar_type_spec( F64_TYPE ); } + | FF64_TYPE { add_scalar_type_spec( FF64_TYPE ); } + | B8_TYPE { add_scalar_type_spec( B8_TYPE ); } + | B16_TYPE { add_scalar_type_spec( B16_TYPE ); } + | B32_TYPE { add_scalar_type_spec( B32_TYPE ); } + | B64_TYPE { add_scalar_type_spec( B64_TYPE ); } + | BB64_TYPE { add_scalar_type_spec( BB64_TYPE ); } + | BB128_TYPE { add_scalar_type_spec( BB128_TYPE ); } + | PRED_TYPE { add_scalar_type_spec( PRED_TYPE ); } + | TEXREF_TYPE { add_scalar_type_spec( TEXREF_TYPE ); } + | SAMPLERREF_TYPE { add_scalar_type_spec( SAMPLERREF_TYPE ); } + | SURFREF_TYPE { add_scalar_type_spec( SURFREF_TYPE ); } + ; + +initializer_list: LEFT_BRACE literal_list RIGHT_BRACE { add_array_initializer(); } + | LEFT_BRACE initializer_list RIGHT_BRACE { syntax_not_implemented(); } + +literal_list: literal_operand + | literal_list COMMA literal_operand; + +instruction_statement: instruction SEMI_COLON + | IDENTIFIER COLON { add_label($1); } + | pred_spec instruction SEMI_COLON; + +instruction: opcode_spec LEFT_PAREN operand RIGHT_PAREN { set_return(); } COMMA operand COMMA LEFT_PAREN operand_list RIGHT_PAREN + | opcode_spec operand COMMA LEFT_PAREN operand_list RIGHT_PAREN + | opcode_spec operand COMMA LEFT_PAREN RIGHT_PAREN + | opcode_spec operand_list + | opcode_spec + ; + +opcode_spec: OPCODE { add_opcode($1); } option_list + | OPCODE { add_opcode($1); } + +pred_spec: PRED IDENTIFIER { add_pred($2,0, -1); } + | PRED EXCLAMATION IDENTIFIER { add_pred($3,1, -1); } + | PRED IDENTIFIER LT_OPTION { add_pred($2,0,1); } + | PRED IDENTIFIER EQ_OPTION { add_pred($2,0,2); } + | PRED IDENTIFIER LE_OPTION { add_pred($2,0,3); } + | PRED IDENTIFIER NE_OPTION { add_pred($2,0,5); } + | PRED IDENTIFIER GE_OPTION { add_pred($2,0,6); } + | PRED IDENTIFIER EQU_OPTION { add_pred($2,0,10); } + | PRED IDENTIFIER GTU_OPTION { add_pred($2,0,12); } + | PRED IDENTIFIER NEU_OPTION { add_pred($2,0,13); } + | PRED IDENTIFIER CF_OPTION { add_pred($2,0,17); } + | PRED IDENTIFIER SF_OPTION { add_pred($2,0,19); } + | PRED IDENTIFIER NSF_OPTION { add_pred($2,0,28); } + ; + +option_list: option + | option option_list ; + +option: type_spec + | compare_spec + | addressable_spec + | rounding_mode + | wmma_spec + | SYNC_OPTION { add_option(SYNC_OPTION); } + | ARRIVE_OPTION { add_option(ARRIVE_OPTION); } + | RED_OPTION { add_option(RED_OPTION); } + | UNI_OPTION { add_option(UNI_OPTION); } + | WIDE_OPTION { add_option(WIDE_OPTION); } + | ANY_OPTION { add_option(ANY_OPTION); } + | ALL_OPTION { add_option(ALL_OPTION); } + | BALLOT_OPTION { add_option(BALLOT_OPTION); } + | GLOBAL_OPTION { add_option(GLOBAL_OPTION); } + | CTA_OPTION { add_option(CTA_OPTION); } + | SYS_OPTION { add_option(SYS_OPTION); } + | GEOM_MODIFIER_1D { add_option(GEOM_MODIFIER_1D); } + | GEOM_MODIFIER_2D { add_option(GEOM_MODIFIER_2D); } + | GEOM_MODIFIER_3D { add_option(GEOM_MODIFIER_3D); } + | SAT_OPTION { add_option(SAT_OPTION); } + | FTZ_OPTION { add_option(FTZ_OPTION); } + | NEG_OPTION { add_option(NEG_OPTION); } + | APPROX_OPTION { add_option(APPROX_OPTION); } + | FULL_OPTION { add_option(FULL_OPTION); } + | EXIT_OPTION { add_option(EXIT_OPTION); } + | ABS_OPTION { add_option(ABS_OPTION); } + | atomic_operation_spec ; + | TO_OPTION { add_option(TO_OPTION); } + | HALF_OPTION { add_option(HALF_OPTION); } + | EXTP_OPTION { add_option(EXTP_OPTION); } + | CA_OPTION { add_option(CA_OPTION); } + | CG_OPTION { add_option(CG_OPTION); } + | CS_OPTION { add_option(CS_OPTION); } + | LU_OPTION { add_option(LU_OPTION); } + | CV_OPTION { add_option(CV_OPTION); } + | WB_OPTION { add_option(WB_OPTION); } + | WT_OPTION { add_option(WT_OPTION); } + | NC_OPTION { add_option(NC_OPTION); } + | UP_OPTION { add_option(UP_OPTION); } + | DOWN_OPTION { add_option(DOWN_OPTION); } + | BFLY_OPTION { add_option(BFLY_OPTION); } + | IDX_OPTION { add_option(IDX_OPTION); } + ; + +atomic_operation_spec: ATOMIC_AND { add_option(ATOMIC_AND); } + | ATOMIC_POPC { add_option(ATOMIC_POPC); } + | ATOMIC_OR { add_option(ATOMIC_OR); } + | ATOMIC_XOR { add_option(ATOMIC_XOR); } + | ATOMIC_CAS { add_option(ATOMIC_CAS); } + | ATOMIC_EXCH { add_option(ATOMIC_EXCH); } + | ATOMIC_ADD { add_option(ATOMIC_ADD); } + | ATOMIC_INC { add_option(ATOMIC_INC); } + | ATOMIC_DEC { add_option(ATOMIC_DEC); } + | ATOMIC_MIN { add_option(ATOMIC_MIN); } + | ATOMIC_MAX { add_option(ATOMIC_MAX); } + ; + +rounding_mode: floating_point_rounding_mode + | integer_rounding_mode; + + +floating_point_rounding_mode: RN_OPTION { add_option(RN_OPTION); } + | RZ_OPTION { add_option(RZ_OPTION); } + | RM_OPTION { add_option(RM_OPTION); } + | RP_OPTION { add_option(RP_OPTION); } + ; + +integer_rounding_mode: RNI_OPTION { add_option(RNI_OPTION); } + | RZI_OPTION { add_option(RZI_OPTION); } + | RMI_OPTION { add_option(RMI_OPTION); } + | RPI_OPTION { add_option(RPI_OPTION); } + ; + +compare_spec:EQ_OPTION { add_option(EQ_OPTION); } + | NE_OPTION { add_option(NE_OPTION); } + | LT_OPTION { add_option(LT_OPTION); } + | LE_OPTION { add_option(LE_OPTION); } + | GT_OPTION { add_option(GT_OPTION); } + | GE_OPTION { add_option(GE_OPTION); } + | LO_OPTION { add_option(LO_OPTION); } + | LS_OPTION { add_option(LS_OPTION); } + | HI_OPTION { add_option(HI_OPTION); } + | HS_OPTION { add_option(HS_OPTION); } + | EQU_OPTION { add_option(EQU_OPTION); } + | NEU_OPTION { add_option(NEU_OPTION); } + | LTU_OPTION { add_option(LTU_OPTION); } + | LEU_OPTION { add_option(LEU_OPTION); } + | GTU_OPTION { add_option(GTU_OPTION); } + | GEU_OPTION { add_option(GEU_OPTION); } + | NUM_OPTION { add_option(NUM_OPTION); } + | NAN_OPTION { add_option(NAN_OPTION); } + ; + +wmma_spec: WMMA_DIRECTIVE LAYOUT CONFIGURATION{add_wmma_option($1);add_wmma_option($2);add_wmma_option($3);} + | WMMA_DIRECTIVE LAYOUT LAYOUT CONFIGURATION{add_wmma_option($1);add_wmma_option($2),add_wmma_option($3),add_wmma_option($4)} + | WMMA_DIRECTIVE LAYOUT CONFIGURATION ptr_space_spec{add_wmma_option($1);add_wmma_option($2),add_wmma_option($3),add_wmma_option($4)} + | WMMA_DIRECTIVE LAYOUT LAYOUT CONFIGURATION ptr_space_spec{add_wmma_option($1);add_wmma_option($2),add_wmma_option($3),add_wmma_option($4)} + ; + +operand_list: operand + | operand COMMA operand_list; + +operand: IDENTIFIER { add_scalar_operand( $1 ); } + | EXCLAMATION IDENTIFIER { add_neg_pred_operand( $2 ); } + | MINUS IDENTIFIER { add_scalar_operand( $2 ); change_operand_neg(); } + | memory_operand + | literal_operand + | builtin_operand + | vector_operand + | MINUS vector_operand { change_operand_neg(); } + | tex_operand + | IDENTIFIER PLUS INT_OPERAND { add_address_operand($1,$3); } + | IDENTIFIER LO_OPTION { add_scalar_operand( $1 ); change_operand_lohi(1);} + | MINUS IDENTIFIER LO_OPTION { add_scalar_operand( $2 ); change_operand_lohi(1); change_operand_neg();} + | IDENTIFIER HI_OPTION { add_scalar_operand( $1 ); change_operand_lohi(2);} + | MINUS IDENTIFIER HI_OPTION { add_scalar_operand( $2 ); change_operand_lohi(2); change_operand_neg();} + | IDENTIFIER PIPE IDENTIFIER { add_2vector_operand($1,$3); change_double_operand_type(-1);} + | IDENTIFIER PIPE IDENTIFIER LO_OPTION { add_2vector_operand($1,$3); change_double_operand_type(-1); change_operand_lohi(1);} + | IDENTIFIER PIPE IDENTIFIER HI_OPTION { add_2vector_operand($1,$3); change_double_operand_type(-1); change_operand_lohi(2);} + | IDENTIFIER BACKSLASH IDENTIFIER { add_2vector_operand($1,$3); change_double_operand_type(-3);} + | IDENTIFIER BACKSLASH IDENTIFIER LO_OPTION { add_2vector_operand($1,$3); change_double_operand_type(-3); change_operand_lohi(1);} + | IDENTIFIER BACKSLASH IDENTIFIER HI_OPTION { add_2vector_operand($1,$3); change_double_operand_type(-3); change_operand_lohi(2);} + ; + +vector_operand: LEFT_BRACE IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { add_2vector_operand($2,$4); } + | LEFT_BRACE IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { add_3vector_operand($2,$4,$6); } + | LEFT_BRACE IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { add_4vector_operand($2,$4,$6,$8); } + | LEFT_BRACE IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { add_8vector_operand($2,$4,$6,$8,$10,$12,$14,$16); } + | LEFT_BRACE IDENTIFIER RIGHT_BRACE { add_1vector_operand($2); } + ; + +tex_operand: LEFT_SQUARE_BRACKET IDENTIFIER COMMA { add_scalar_operand($2); } + vector_operand + RIGHT_SQUARE_BRACKET + ; + +builtin_operand: SPECIAL_REGISTER DIMENSION_MODIFIER { add_builtin_operand($1,$2); } + | SPECIAL_REGISTER { add_builtin_operand($1,-1); } + ; + +memory_operand : LEFT_SQUARE_BRACKET address_expression RIGHT_SQUARE_BRACKET { add_memory_operand(); } + | IDENTIFIER LEFT_SQUARE_BRACKET address_expression RIGHT_SQUARE_BRACKET { add_memory_operand(); change_memory_addr_space($1); } + | IDENTIFIER LEFT_SQUARE_BRACKET literal_operand RIGHT_SQUARE_BRACKET { change_memory_addr_space($1); } + | IDENTIFIER LEFT_SQUARE_BRACKET twin_operand RIGHT_SQUARE_BRACKET { change_memory_addr_space($1); add_memory_operand();} + | MINUS memory_operand { change_operand_neg(); } + ; + +twin_operand : IDENTIFIER PLUS IDENTIFIER { add_double_operand($1,$3); change_double_operand_type(1); } + | IDENTIFIER PLUS IDENTIFIER LO_OPTION { add_double_operand($1,$3); change_double_operand_type(1); change_operand_lohi(1); } + | IDENTIFIER PLUS IDENTIFIER HI_OPTION { add_double_operand($1,$3); change_double_operand_type(1); change_operand_lohi(2); } + | IDENTIFIER PLUS EQUALS IDENTIFIER { add_double_operand($1,$4); change_double_operand_type(2); } + | IDENTIFIER PLUS EQUALS IDENTIFIER LO_OPTION { add_double_operand($1,$4); change_double_operand_type(2); change_operand_lohi(1); } + | IDENTIFIER PLUS EQUALS IDENTIFIER HI_OPTION { add_double_operand($1,$4); change_double_operand_type(2); change_operand_lohi(2); } + | IDENTIFIER PLUS EQUALS INT_OPERAND { add_address_operand($1,$4); change_double_operand_type(3); } + ; + +literal_operand : INT_OPERAND { add_literal_int($1); } + | FLOAT_OPERAND { add_literal_float($1); } + | DOUBLE_OPERAND { add_literal_double($1); } + ; + +address_expression: IDENTIFIER { add_address_operand($1,0); } + | IDENTIFIER LO_OPTION { add_address_operand($1,0); change_operand_lohi(1);} + | IDENTIFIER HI_OPTION { add_address_operand($1,0); change_operand_lohi(2); } + | IDENTIFIER PLUS INT_OPERAND { add_address_operand($1,$3); } + | INT_OPERAND { add_address_operand2($1); } + ; + +%% + +extern int ptx_lineno; +extern const char *g_filename; + +void syntax_not_implemented() +{ + printf("Parse error (%s:%u): this syntax is not (yet) implemented:\n",g_filename,ptx_lineno); + ptx_error(NULL); + abort(); +} diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 833f175..16cc975 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1078,7 +1078,7 @@ public: int membar_level() const { return m_membar_level; } bool has_memory_read() const { - if( m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP ) + if( m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP|| m_opcode==MMA_LD_OP) return true; // Check PTXPlus operand type below // Source operands are memory operands @@ -1090,7 +1090,7 @@ public: return false; } bool has_memory_write() const { - if( m_opcode == ST_OP ) return true; + if( m_opcode == ST_OP || m_opcode==MMA_ST_OP ) return true; // Check PTXPlus operand type below // Destination operand is a memory operand ptx_instruction::const_iterator op=op_iter_begin(); diff --git a/src/cuda-sim/ptx_sim.h b/src/cuda-sim/ptx_sim.h index 05acf20..403ce5b 100644 --- a/src/cuda-sim/ptx_sim.h +++ b/src/cuda-sim/ptx_sim.h @@ -303,6 +303,15 @@ public: const ptx_reg_t &data2, const ptx_reg_t &data3, const ptx_reg_t &data4 ); + void set_wmma_vector_operand_values( const operand_info &dst, + const ptx_reg_t &data1, + const ptx_reg_t &data2, + const ptx_reg_t &data3, + const ptx_reg_t &data4, + const ptx_reg_t &data5, + const ptx_reg_t &data6, + const ptx_reg_t &data7, + const ptx_reg_t &data8 ); function_info *func_info() { diff --git a/src/gpgpu-sim/Makefile b/src/gpgpu-sim/Makefile index f10a8a4..4f77699 100644 --- a/src/gpgpu-sim/Makefile +++ b/src/gpgpu-sim/Makefile @@ -48,7 +48,7 @@ ifeq ($(GNUC_CPP0X), 1) endif ifneq ($(DEBUG),1) - OPTFLAGS += -O3 + OPTFLAGS += -O0 else CXXFLAGS += endif diff --git a/src/intersim2/Makefile b/src/intersim2/Makefile index bd42000..4ef21ac 100644 --- a/src/intersim2/Makefile +++ b/src/intersim2/Makefile @@ -44,7 +44,7 @@ endif CPPFLAGS += -Wall $(INCPATH) $(DEFINE) ifneq ($(DEBUG),1) -CPPFLAGS += -O3 +CPPFLAGS += -O0 endif CPPFLAGS += -g CPPFLAGS += -fPIC -- cgit v1.3 From 03de5ae03420ba5666d669c6f76faccf2704fa58 Mon Sep 17 00:00:00 2001 From: aamir Date: Thu, 31 May 2018 00:48:37 -0700 Subject: mma_ld_impl --- src/cuda-sim/ptx.y | 2 +- src/cuda-sim/ptx_ir.h | 2 +- src/cuda-sim/ptx_parser.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/src/cuda-sim/ptx.y b/src/cuda-sim/ptx.y index 737657c..012451c 100644 --- a/src/cuda-sim/ptx.y +++ b/src/cuda-sim/ptx.y @@ -520,7 +520,7 @@ compare_spec:EQ_OPTION { add_option(EQ_OPTION); } | NAN_OPTION { add_option(NAN_OPTION); } ; -wmma_spec: WMMA_DIRECTIVE LAYOUT CONFIGURATION{add_wmma_option($1);add_wmma_option($2);add_wmma_option($3);} +wmma_spec: WMMA_DIRECTIVE LAYOUT CONFIGURATION{add_space_spec(global_space,0);add_ptr_spec(global_space); add_wmma_option($1);add_wmma_option($2);add_wmma_option($3);} | WMMA_DIRECTIVE LAYOUT LAYOUT CONFIGURATION{add_wmma_option($1);add_wmma_option($2),add_wmma_option($3),add_wmma_option($4)} ; diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 16cc975..6bba717 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -665,7 +665,7 @@ public: const symbol* vec_symbol(int idx) const { - assert(idx < 4); + assert(idx < 8); const symbol *result = m_value.m_vector_symbolic[idx]; assert( result != NULL ); return result; diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index 6757091..eb81961 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -39,7 +39,7 @@ void set_ptx_warp_size(const struct core_config * warp_size) g_shader_core_config=warp_size; } -static bool g_debug_ir_generation=false; +static bool g_debug_ir_generation=true; const char *g_filename; unsigned g_max_regs_per_thread = 0; -- cgit v1.3 From 4161ccba0d4a99157afed3cdccef0e9c2a6d89e6 Mon Sep 17 00:00:00 2001 From: aamir Date: Tue, 5 Jun 2018 12:50:57 -0700 Subject: added support for wmma:load_c:f16_type --- cuda-kernels/tensorcore_type16_16.cu | 217 ++++++++++++++++++++++++++ cuda-kernels/tensorcore_type32_16.cu | 218 ++++++++++++++++++++++++++ cuda-kernels/tensorcore_type32_32.cu | 15 ++ src/cuda-sim/instructions.cc | 287 +++++++++++++++++++++++++---------- src/cuda-sim/ptx_ir.cc | 21 +++ src/cuda-sim/ptx_ir.h | 9 +- 6 files changed, 686 insertions(+), 81 deletions(-) create mode 100644 cuda-kernels/tensorcore_type16_16.cu create mode 100644 cuda-kernels/tensorcore_type32_16.cu (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/cuda-kernels/tensorcore_type16_16.cu b/cuda-kernels/tensorcore_type16_16.cu new file mode 100644 index 0000000..2b93bf5 --- /dev/null +++ b/cuda-kernels/tensorcore_type16_16.cu @@ -0,0 +1,217 @@ +#include +#include + +// Define some error checking macros. +#define cudaErrCheck(stat) { cudaErrCheck_((stat), __FILE__, __LINE__); } +void cudaErrCheck_(cudaError_t stat, const char *file, int line) { + if (stat != cudaSuccess) { + fprintf(stderr, "CUDA Error: %s %s %d\n", cudaGetErrorString(stat), file, line); + } +} + +#define curandErrCheck(stat) { curandErrCheck_((stat), __FILE__, __LINE__); } +void curandErrCheck_(curandStatus_t stat, const char *file, int line) { + if (stat != CURAND_STATUS_SUCCESS) { + fprintf(stderr, "cuRand Error: %d %s %d\n", stat, file, line); + } +} + +#include +using namespace nvcuda; + +// Must be multiples of 16 for wmma code to work +#define MATRIX_M (16) +#define MATRIX_N (16) +#define MATRIX_K (16) + + +// The only dimensions currently supported by WMMA +const int WMMA_M = 16; +const int WMMA_N = 16; +const int WMMA_K = 16; + +__global__ void wmma_example(half *a, half *b, half *c,half *d_fp16, int M, int N, int K) { + //unsigned int start_time=0,end_time=0; + //start_time=clock(); + + // Declare the fragments + wmma::fragment a_frag; + wmma::fragment b_frag; + wmma::fragment c_frag; + + // Bounds checking + wmma::load_matrix_sync(a_frag, a, K); + wmma::load_matrix_sync(b_frag, b, K); + wmma::load_matrix_sync(c_frag, c, N,wmma::mem_col_major); + wmma::mma_sync(c_frag, a_frag, b_frag, c_frag); + + wmma::store_matrix_sync(d_fp16, c_frag, N, wmma::mem_col_major); + //printf("clock=%d",end_time-start_time); +} + +__global__ void convertFp32ToFp16 (half *out, float *in, int n) { + int idx = blockDim.x * blockIdx.x + threadIdx.x; + if (idx < n) { + out[idx] = in[idx]; + } +} +__global__ void convertFp16ToFp32 (float *out, half *in, int n) { + int idx = blockDim.x * blockIdx.x + threadIdx.x; + if (idx < n) { + out[idx] = in[idx]; + } +} + +int main(int argc, char* argv[]) { + float *a_fp32; + float *b_fp32; + float *c_fp32; + float *d_fp32; + + half *a_fp16; + half *b_fp16; + half *c_fp16; + half *d_fp16; + + float *a_host_wmma; + float *b_host_wmma; + float *c_host_wmma; + float *d_host_wmma; + float *d_cal_host_wmma; + + cudaEvent_t startWMMA; + cudaEvent_t stopWMMA; + + + cudaErrCheck(cudaEventCreate(&startWMMA)); + cudaErrCheck(cudaEventCreate(&stopWMMA)); + + // Use tensor cores + cudaErrCheck(cudaMalloc((void**)&a_fp32, MATRIX_M * MATRIX_K * sizeof(float))); + cudaErrCheck(cudaMalloc((void**)&b_fp32, MATRIX_K * MATRIX_N * sizeof(float))); + cudaErrCheck(cudaMalloc((void**)&c_fp32, MATRIX_K * MATRIX_N * sizeof(float))); + cudaErrCheck(cudaMalloc((void**)&d_fp32, MATRIX_K * MATRIX_N * sizeof(float))); + cudaErrCheck(cudaMalloc((void**)&a_fp16, MATRIX_M * MATRIX_K * sizeof(half))); + cudaErrCheck(cudaMalloc((void**)&b_fp16, MATRIX_K * MATRIX_N * sizeof(half))); + cudaErrCheck(cudaMalloc((void**)&c_fp16, MATRIX_K * MATRIX_N * sizeof(half))); + cudaErrCheck(cudaMalloc((void**)&d_fp16, MATRIX_K * MATRIX_N * sizeof(half))); + + + a_host_wmma = (float*)malloc(MATRIX_M * MATRIX_K * sizeof(float)); + b_host_wmma = (float*)malloc(MATRIX_K * MATRIX_N * sizeof(float)); + c_host_wmma = (float*)malloc(MATRIX_M * MATRIX_N * sizeof(float)); + d_host_wmma = (float*)malloc(MATRIX_M * MATRIX_N * sizeof(float)); + d_cal_host_wmma = (float*)malloc(MATRIX_M * MATRIX_N * sizeof(float)); + + //printf("a_fp32\n"); + for(int m=0;m>> (a_fp16, a_fp32, MATRIX_M * MATRIX_K); + convertFp32ToFp16 <<< (MATRIX_K * MATRIX_N + 255) / 256, 256 >>> (b_fp16, b_fp32, MATRIX_K * MATRIX_N); + convertFp32ToFp16 <<< (MATRIX_M * MATRIX_N + 255) / 256, 256 >>> (c_fp16, c_fp32, MATRIX_K * MATRIX_N); + + printf("\nM = %d, N = %d, K = %d. \n", MATRIX_M, MATRIX_N, MATRIX_K); + + printf("Running with wmma...\n"); + cudaErrCheck(cudaEventRecord(startWMMA)); + wmma_example <<< 1, 32>>> (a_fp16, b_fp16, c_fp16, d_fp16 , MATRIX_M, MATRIX_N, MATRIX_K); + cudaErrCheck(cudaEventRecord(stopWMMA)); + cudaErrCheck(cudaEventSynchronize(stopWMMA)); + + convertFp16ToFp32 <<< (MATRIX_M * MATRIX_N + 255) / 256, 256 >>> (d_fp32, d_fp16, MATRIX_K * MATRIX_N); + // Error checking + printf("\nChecking results...\n"); + cudaErrCheck(cudaMemcpy(d_host_wmma, d_fp32, MATRIX_M * MATRIX_N * sizeof(float), cudaMemcpyDeviceToHost)); + + printf("Results verified: cublas and WMMA agree.\n\n"); + float wmmaTime; + cudaErrCheck(cudaEventElapsedTime(&wmmaTime, startWMMA, stopWMMA)); + printf("wmma took %fms\n", wmmaTime); + + cudaErrCheck(cudaEventDestroy(startWMMA)); + cudaErrCheck(cudaEventDestroy(stopWMMA)); + + int t=200000; + while(t-->0); + printf("D_CALCULATED\n"); + for(int m=0;m1) + { + printf("ERROR:\n"); + suc=0; + } + } + } + if(suc==1) + printf("COMPLETED_SUCCESSFULLY\n"); + + cudaErrCheck(cudaFree(a_fp32)); + cudaErrCheck(cudaFree(b_fp32)); + cudaErrCheck(cudaFree(c_fp32)); + cudaErrCheck(cudaFree(d_fp32)); + cudaErrCheck(cudaFree(a_fp16)); + cudaErrCheck(cudaFree(b_fp16)); + cudaErrCheck(cudaFree(c_fp16)); + cudaErrCheck(cudaFree(d_fp16)); + + free(a_host_wmma); + free(b_host_wmma); + free(c_host_wmma); + free(d_host_wmma); + cudaErrCheck(cudaDeviceReset()); + return 0; +} + + diff --git a/cuda-kernels/tensorcore_type32_16.cu b/cuda-kernels/tensorcore_type32_16.cu new file mode 100644 index 0000000..c66d8f8 --- /dev/null +++ b/cuda-kernels/tensorcore_type32_16.cu @@ -0,0 +1,218 @@ +#include +#include + +// Define some error checking macros. +#define cudaErrCheck(stat) { cudaErrCheck_((stat), __FILE__, __LINE__); } +void cudaErrCheck_(cudaError_t stat, const char *file, int line) { + if (stat != cudaSuccess) { + fprintf(stderr, "CUDA Error: %s %s %d\n", cudaGetErrorString(stat), file, line); + } +} + +#define curandErrCheck(stat) { curandErrCheck_((stat), __FILE__, __LINE__); } +void curandErrCheck_(curandStatus_t stat, const char *file, int line) { + if (stat != CURAND_STATUS_SUCCESS) { + fprintf(stderr, "cuRand Error: %d %s %d\n", stat, file, line); + } +} + +#include +using namespace nvcuda; + +// Must be multiples of 16 for wmma code to work +#define MATRIX_M (16) +#define MATRIX_N (16) +#define MATRIX_K (16) + + +// The only dimensions currently supported by WMMA +const int WMMA_M = 16; +const int WMMA_N = 16; +const int WMMA_K = 16; + +__global__ void wmma_example(half *a, half *b, half *c,float *d_fp32, int M, int N, int K) { + //unsigned int start_time=0,end_time=0; + //start_time=clock(); + + // Declare the fragments + wmma::fragment a_frag; + wmma::fragment b_frag; + wmma::fragment c_frag; + wmma::fragment d_frag; + + // Bounds checking + wmma::load_matrix_sync(a_frag, a, K); + wmma::load_matrix_sync(b_frag, b, K); + wmma::load_matrix_sync(c_frag, c, N,wmma::mem_col_major); + wmma::mma_sync(d_frag, a_frag, b_frag, c_frag); + + wmma::store_matrix_sync(d_fp32, d_frag, N, wmma::mem_col_major); + //printf("clock=%d",end_time-start_time); +} + +__global__ void convertFp32ToFp16 (half *out, float *in, int n) { + int idx = blockDim.x * blockIdx.x + threadIdx.x; + if (idx < n) { + out[idx] = in[idx]; + } +} +__global__ void convertFp16ToFp32 (float *out, half *in, int n) { + int idx = blockDim.x * blockIdx.x + threadIdx.x; + if (idx < n) { + out[idx] = in[idx]; + } +} + +int main(int argc, char* argv[]) { + float *a_fp32; + float *b_fp32; + float *c_fp32; + float *d_fp32; + + half *a_fp16; + half *b_fp16; + half *c_fp16; + half *d_fp16; + + float *a_host_wmma; + float *b_host_wmma; + float *c_host_wmma; + float *d_host_wmma; + float *d_cal_host_wmma; + + cudaEvent_t startWMMA; + cudaEvent_t stopWMMA; + + + cudaErrCheck(cudaEventCreate(&startWMMA)); + cudaErrCheck(cudaEventCreate(&stopWMMA)); + + // Use tensor cores + cudaErrCheck(cudaMalloc((void**)&a_fp32, MATRIX_M * MATRIX_K * sizeof(float))); + cudaErrCheck(cudaMalloc((void**)&b_fp32, MATRIX_K * MATRIX_N * sizeof(float))); + cudaErrCheck(cudaMalloc((void**)&c_fp32, MATRIX_K * MATRIX_N * sizeof(float))); + cudaErrCheck(cudaMalloc((void**)&d_fp32, MATRIX_K * MATRIX_N * sizeof(float))); + cudaErrCheck(cudaMalloc((void**)&a_fp16, MATRIX_M * MATRIX_K * sizeof(half))); + cudaErrCheck(cudaMalloc((void**)&b_fp16, MATRIX_K * MATRIX_N * sizeof(half))); + cudaErrCheck(cudaMalloc((void**)&c_fp16, MATRIX_K * MATRIX_N * sizeof(half))); + cudaErrCheck(cudaMalloc((void**)&d_fp16, MATRIX_K * MATRIX_N * sizeof(half))); + + + a_host_wmma = (float*)malloc(MATRIX_M * MATRIX_K * sizeof(float)); + b_host_wmma = (float*)malloc(MATRIX_K * MATRIX_N * sizeof(float)); + c_host_wmma = (float*)malloc(MATRIX_M * MATRIX_N * sizeof(float)); + d_host_wmma = (float*)malloc(MATRIX_M * MATRIX_N * sizeof(float)); + d_cal_host_wmma = (float*)malloc(MATRIX_M * MATRIX_N * sizeof(float)); + + //printf("a_fp32\n"); + for(int m=0;m>> (a_fp16, a_fp32, MATRIX_M * MATRIX_K); + convertFp32ToFp16 <<< (MATRIX_K * MATRIX_N + 255) / 256, 256 >>> (b_fp16, b_fp32, MATRIX_K * MATRIX_N); + convertFp32ToFp16 <<< (MATRIX_M * MATRIX_N + 255) / 256, 256 >>> (c_fp16, c_fp32, MATRIX_K * MATRIX_N); + + printf("\nM = %d, N = %d, K = %d. \n", MATRIX_M, MATRIX_N, MATRIX_K); + + printf("Running with wmma...\n"); + cudaErrCheck(cudaEventRecord(startWMMA)); + wmma_example <<< 1, 32>>> (a_fp16, b_fp16, c_fp16, d_fp32 , MATRIX_M, MATRIX_N, MATRIX_K); + cudaErrCheck(cudaEventRecord(stopWMMA)); + cudaErrCheck(cudaEventSynchronize(stopWMMA)); + + //convertFp16ToFp32 <<< (MATRIX_M * MATRIX_N + 255) / 256, 256 >>> (d_fp32, d_fp16, MATRIX_K * MATRIX_N); + // Error checking + printf("\nChecking results...\n"); + cudaErrCheck(cudaMemcpy(d_host_wmma, d_fp32, MATRIX_M * MATRIX_N * sizeof(float), cudaMemcpyDeviceToHost)); + + printf("Results verified: cublas and WMMA agree.\n\n"); + float wmmaTime; + cudaErrCheck(cudaEventElapsedTime(&wmmaTime, startWMMA, stopWMMA)); + printf("wmma took %fms\n", wmmaTime); + + cudaErrCheck(cudaEventDestroy(startWMMA)); + cudaErrCheck(cudaEventDestroy(stopWMMA)); + + int t=600000; + while(t-->0); + printf("D_CALCULATED\n"); + for(int m=0;m1) + { + printf("ERROR:\n"); + suc=0; + } + } + } + if(suc==1) + printf("COMPLETED_SUCCESSFULLY\n"); + + cudaErrCheck(cudaFree(a_fp32)); + cudaErrCheck(cudaFree(b_fp32)); + cudaErrCheck(cudaFree(c_fp32)); + cudaErrCheck(cudaFree(d_fp32)); + cudaErrCheck(cudaFree(a_fp16)); + cudaErrCheck(cudaFree(b_fp16)); + cudaErrCheck(cudaFree(c_fp16)); + cudaErrCheck(cudaFree(d_fp16)); + + free(a_host_wmma); + free(b_host_wmma); + free(c_host_wmma); + free(d_host_wmma); + cudaErrCheck(cudaDeviceReset()); + return 0; +} + + diff --git a/cuda-kernels/tensorcore_type32_32.cu b/cuda-kernels/tensorcore_type32_32.cu index 0d26163..73386f9 100644 --- a/cuda-kernels/tensorcore_type32_32.cu +++ b/cuda-kernels/tensorcore_type32_32.cu @@ -167,7 +167,10 @@ int main(int argc, char* argv[]) { cudaErrCheck(cudaEventDestroy(startWMMA)); cudaErrCheck(cudaEventDestroy(stopWMMA)); + int t=200000; + while(t-->0); printf("D_CALCULATED\n"); + for(int m=0;m1) + { + printf("ERROR:\n"); + suc=0; + } + } + } + if(suc==1) + printf("COMPLETED_SUCCESSFULLY\n"); cudaErrCheck(cudaFree(a_fp32)); cudaErrCheck(cudaFree(b_fp32)); diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 70aee35..f314e62 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -57,6 +57,42 @@ const char *g_opcode_string[NUM_OPCODES] = { #undef OP_W_DEF }; +unsigned thread_group_offset(int thread){ + unsigned thread_group=thread/4; + unsigned in_tg_index=thread%4; + unsigned offset; + switch(thread_group){ + case 0: + offset=0; + break; + case 1: + offset=8; + break; + + case 2: + offset=128; + break; + case 3: + offset=136; + break; + case 4: + offset=4; + break; + case 5: + offset=12; + break; + case 6: + offset=132; + break; + case 7: + offset=140; + break; + default: + abort(); + + } + return offset+in_tg_index; +} void inst_not_implemented( const ptx_instruction * pI ) ; ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread); @@ -655,7 +691,7 @@ void ptx_thread_info::set_wmma_vector_operand_values( const operand_info &dst, const ptx_reg_t &data8 ) { unsigned num_elements = dst.get_vect_nelem(); - if (num_elements > 7) { + if (num_elements == 8) { set_reg(dst.vec_symbol(0), data1); set_reg(dst.vec_symbol(1), data2); set_reg(dst.vec_symbol(2), data3); @@ -1534,6 +1570,7 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) ptx_reg_t src_data; ptx_thread_info *thread; + unsigned wmma_type = pI->get_wmma_type(); unsigned type = pI->get_type(); unsigned type2 = pI->get_type2(); int tid = inst.warp_id_func() * core->get_warp_size(); @@ -1543,9 +1580,8 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) //NOT WOR const operand_info &src_a= pI->operand_lookup(1); //NOT WOR src_data= (thread->get_operand_value(src_a, dst, type, thread, 1)); //NOT WOR thread->set_operand_value(dst, src_data, type, thread, pI); + unsigned thread_group_index; for (thrd=0; thrd < core->get_warp_size(); thrd++){ - row=thrd/2; - offset=8*(thrd%2); thread = core->get_thread_info()[tid+thrd]; printf("thread=%d:",thrd); for(i=1;i<=3;i++){ @@ -1554,40 +1590,84 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) unsigned nelem = src_a.get_vect_nelem(); ptx_reg_t v[8]; thread->get_vector_operand_values( src_a, v, nelem ); - if(i!=3||((i==3)&&(type==F16_TYPE))){ - printf("%x ",v[0].f16); - printf("%x ",v[1].f16); - printf("%x ",v[2].f16); - printf("%x ",v[3].f16); - printf("%x ",v[4].f16); - printf("%x ",v[5].f16); - printf("%x ",v[6].f16); - printf("%x ",v[7].f16); + if(i!=3){ + printf("%x ",v[0].f16); + printf("%x ",v[1].f16); + printf("%x ",v[2].f16); + printf("%x ",v[3].f16); + printf("%x ",v[4].f16); + printf("%x ",v[5].f16); + printf("%x ",v[6].f16); + printf("%x ",v[7].f16); + } else{ - printf("%f ",v[0].f32); - printf("%f ",v[1].f32); - printf("%f ",v[2].f32); - printf("%f ",v[3].f32); - printf("%f ",v[4].f32); - printf("%f ",v[5].f32); - printf("%f ",v[6].f32); - printf("%f ",v[7].f32); + if(type2==F32_TYPE){ + printf("%f ",v[0].f32); + printf("%f ",v[1].f32); + printf("%f ",v[2].f32); + printf("%f ",v[3].f32); + printf("%f ",v[4].f32); + printf("%f ",v[5].f32); + printf("%f ",v[6].f32); + printf("%f ",v[7].f32); + } + else{ + printf("%x ",v[0].s64); + printf("%x ",v[1].s64); + printf("%x ",v[2].s64); + printf("%x ",v[3].s64); + } } - + thread_group_index=thread_group_offset(thrd); + row=(thread_group_index/16); + offset=thread_group_index%16; switch(i) { case 1 ://operand 1 for(k=0;k<8;k++) - matrix_a[row][offset+k]=v[k]; + matrix_a[row+k][offset]=v[k]; break; case 2 ://operand 2 for(k=0;k<8;k++) - matrix_b[row][offset+k]=v[k]; + matrix_b[row+k][offset]=v[k]; break; case 3 ://operand 3 - for(k=0;k<8;k++) - matrix_c[row][offset+k]=v[k]; - break; + if(type2!=F16_TYPE){ + for(k=0;k<8;k++) + matrix_c[row+k][offset]=v[k]; + } + else { + ptx_reg_t nw_v[8]; + unsigned int n = 0x41933333; + float f = *((float*)&n); + int hex_val; + + for(k=0;k<8;k++){ + if(k%2==0) + hex_val=((v[k/2].s64&0xffff0000)>>16); + else + hex_val=(v[k/2].s64&0xffff); + nw_v[k].f16 =*((half *)&hex_val); + matrix_c[row+k][offset]=nw_v[k]; + } + printf("%x ",nw_v[0].f16); + printf("%x ",nw_v[1].f16); + printf("%x ",nw_v[2].f16); + printf("%x ",nw_v[3].f16); + printf("%x ",nw_v[4].f16); + printf("%x ",nw_v[5].f16); + printf("%x ",nw_v[6].f16); + printf("%x ",nw_v[7].f16); + //float t; + //int m; + //printf("\n"); + //for(m=0;m<8;m++){ + // t=nw_v[m].f16; + // printf(" %f ",t); + //} + //printf("\n"); + } + break; default : printf("Invalid Operand Index\n" ); } @@ -1612,10 +1692,10 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) printf("MATRIX_C\n"); for (i=0;i<16;i++){ for(j=0;j<16;j++){ - if(type==F16_TYPE) + if(type2==F16_TYPE) printf("%x ",matrix_c[i][j].f16); else - printf("%f ",matrix_c[i][j].f32); + printf("%f ",matrix_c[i][j].f32); } printf("\n"); } @@ -1628,18 +1708,33 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) printf("MATRIX_D\n"); for (i=0;i<16;i++){ for(j=0;j<16;j++){ - printf("%x ",matrix_d[i][j].f16); + if(type==F16_TYPE) + printf("%x ",matrix_d[i][j].f16); + else + printf("%.2f ",matrix_d[i][j].f32); + } printf("\n"); } float temp; + half temp2; for (i=0;i<16;i++){ for(j=0;j<16;j++){ for(k=0;k<16;k++){ matrix_d[i][j].f16=matrix_d[i][j].f16+matrix_a[i][k].f16*matrix_b[k][j].f16; } - if(type==F16_TYPE) + if((type==F16_TYPE)&&(type2==F16_TYPE)) matrix_d[i][j].f16+=matrix_c[i][j].f16; + else if((type==F32_TYPE)&&(type2==F16_TYPE)){ + temp2=matrix_d[i][j].f16+matrix_c[i][j].f16; + temp=temp2; + matrix_d[i][j].f32=temp; + } + else if((type==F16_TYPE)&&(type2==F32_TYPE)){ + temp=matrix_d[i][j].f16; + temp+=matrix_c[i][j].f32; + matrix_d[i][j].f16=half(temp); + } else{ temp=matrix_d[i][j].f16; temp+=matrix_c[i][j].f32; @@ -1658,16 +1753,33 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) printf("\n"); } for (thrd=0; thrd < core->get_warp_size(); thrd++){ + thread_group_index=thread_group_offset(thrd); + row=(thread_group_index/16); + offset=thread_group_index%16; thread = core->get_thread_info()[tid+thrd]; - row=thrd/2; - offset=8*(thrd%2); - //r2=dst.get_symbol(); - //printf("thrd=%d,i=%d,register%s, data=%f\n",thrd,i,(r2->name()).c_str(),matrix_d[row][offset+i].f32); - //thread->set_operand_value(dst, matrix_d[row][offset+i], type, thread, pI); - thread->set_wmma_vector_operand_values(dst,matrix_d[row][offset],matrix_d[row][offset+1],matrix_d[row][offset+2],matrix_d[row][offset+3],matrix_d[row][offset+4],matrix_d[row][offset+5],matrix_d[row][offset+6],matrix_d[row][offset+7]); - printf("thread%d=%x,%x,%x,%x",thrd,matrix_d[row][offset].f16,matrix_d[row][offset+1].f16,matrix_d[row][offset+2].f16,matrix_d[row][offset+3].f16); - printf(",%x,%x,%x,%x\n",matrix_d[row][offset+4].f16,matrix_d[row][offset+5].f16,matrix_d[row][offset+6].f16,matrix_d[row][offset+7].f16); - } + //r2=dst.get_symbol(); + //printf("thrd=%d,i=%d,register%s, data=%f\n",thrd,i,(r2->name()).c_str(),matrix_d[row][offset+i].f32); + //thread->set_operand_value(dst, matrix_d[row][offset+i], type, thread, pI); + if(type==F32_TYPE){ + thread->set_wmma_vector_operand_values(dst,matrix_d[row][offset],matrix_d[row+1][offset],matrix_d[row+2][offset],matrix_d[row+3][offset],matrix_d[row+4][offset],matrix_d[row+5][offset],matrix_d[row+6][offset],matrix_d[row+7][offset]); + printf("thread%d=%x,%x,%x,%x",thrd,matrix_d[row][offset].f16,matrix_d[row+1][offset].f16,matrix_d[row+2][offset].f16,matrix_d[row+3][offset].f16); + printf(",%x,%x,%x,%x\n",matrix_d[row+4][offset].f16,matrix_d[row+5][offset].f16,matrix_d[row+6][offset].f16,matrix_d[row+7][offset].f16); + } + else if(type==F16_TYPE){ + ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4; + nw_data1.s64=((matrix_d[row][offset].s64 & 0xffff)<<16)|((matrix_d[row+1][offset].s64&0xffff)); + nw_data2.s64=((matrix_d[row+2][offset].s64 & 0xffff)<<16)|((matrix_d[row+3][offset].s64&0xffff)); + nw_data3.s64=((matrix_d[row+4][offset].s64 & 0xffff)<<16)|((matrix_d[row+5][offset].s64&0xffff)); + nw_data4.s64=((matrix_d[row+6][offset].s64 & 0xffff)<<16)|((matrix_d[row+7][offset].s64&0xffff)); + thread->set_vector_operand_values(dst,nw_data1,nw_data2,nw_data3,nw_data4); + printf("thread%d=%x,%x,%x,%x",thrd,nw_data1.s64,nw_data2.s64,nw_data3.s64,nw_data4.s64); + + } + else{ + printf("wmma:mma:wrong type\n"); + abort(); + } + } } void call_impl( const ptx_instruction *pI, ptx_thread_info *thread ) @@ -1910,7 +2022,7 @@ ptx_reg_t f2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, { half mytemp; float myfloat; - assert( from_width == 32); + //assert( from_width == 32); enum cuda_math::cudaRoundMode mode = cuda_math::cudaRoundZero; switch (rounding_mode) { @@ -1959,7 +2071,10 @@ ptx_reg_t f2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, //y.f16 = half(x.f32); printf("f2x: %f\n",myfloat); break; - case 32: assert(0); break; // handled by f2f + case 32: + y.f32=float(x.f16); + + break; // handled by f2f case 64: y.f64 = x.f32; break; @@ -2673,6 +2788,7 @@ void ldu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { ld_exec(pI,thread); } + void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) { size_t size; @@ -2685,6 +2801,7 @@ void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) const operand_info &src2 = pI->operand_lookup(2); int tid = inst.warp_id_func()*core->get_warp_size(); unsigned type = pI->get_type(); + unsigned wmma_type = pI->get_wmma_type(); for (thrd=0; thrd < core->get_warp_size(); thrd++) { thread = core->get_thread_info()[tid+thrd]; @@ -2706,23 +2823,28 @@ void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) type_info_key::type_decode(type,size,t); printf("mma_st: thrd=%d,addr=%d, fp(size=%d), stride=%d\n",thrd,addr_reg.u32,size,src2_data.u32); - if(type==F16_TYPE){ - for(k=0;k<8;k++){ - mem->write(addr+inx*2*src2_data.u32+odd*16+k*size/8,size/8,&v[k].s64,thread,pI); + addr_t new_addr = addr+thread_group_offset(thrd)*size/8; + + ptx_reg_t nw_v[8]; + for(k=0;k<8;k++){ + if(k%2==0) + nw_v[k].s64=((v[k/2].s64&0xffff0000)>>16); + else + nw_v[k].s64=(v[k/2].s64&0xffff); + } + + for(k=0;k<8;k++){ + if(type==F32_TYPE){ + mem->write(new_addr+k*2*size,size/8,&v[k].s64,thread,pI); + printf("wmma:store:thread%d=%x,%x,%x,%x,%x,%x,%x,%x\n",thrd,v[0].s64,v[1].s64,v[2].s64,v[3].s64,v[4].s64,v[5].s64,v[6].s64,v[7].s64); } - } - else if(type==F32_TYPE){ - for(k=0;k<8;k++){ - mem->write(addr+inx*4*src2_data.u32+odd*32+k*size/8,size/8,&v[k].s64,thread,pI); + else if(type==F16_TYPE){ + mem->write(new_addr+k*2*size,size/8,&nw_v[k].s64,thread,pI); + printf("wmma:store:thread%d=%x,%x,%x,%x,%x,%x,%x,%x\n",thrd,nw_v[0].s64,nw_v[1].s64,nw_v[2].s64,nw_v[3].s64,nw_v[4].s64,nw_v[5].s64,nw_v[6].s64,nw_v[7].s64); } - } - else{ - printf("wmma:wrong error type\n"); - } - printf("wmma:store:thread%d=%x,%x,%x,%x,%x,%x,%x,%x\n",thrd,v[0].s64,v[1].s64,v[2].s64,v[3].s64,v[4].s64,v[5].s64,v[6].s64,v[7].s64); - - delete [] v; + + delete [] v; thread->m_last_effective_address = addr; thread->m_last_memory_space = space; } @@ -2736,6 +2858,7 @@ void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) const operand_info &src2 = pI->src2(); unsigned type = pI->get_type(); + unsigned wmma_type = pI->get_wmma_type(); int tid = inst.warp_id_func()*core->get_warp_size(); int thrd,odd,inx; @@ -2758,32 +2881,40 @@ void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) ptx_reg_t data1, data2, data3, data4; ptx_reg_t data5, data6, data7, data8; printf("mma_ld: thrd=%d,addr=%d, fp16(size=%d), stride=%d\n",thrd,src1_data.u32,size,src2_data.u32); - if(type==F16_TYPE){ - mem->read(addr+inx*2*src2_data.u32+odd*16,size/8,&data1.s64); - mem->read(addr+inx*2*src2_data.u32+odd*16+size/8,size/8,&data2.s64); - mem->read(addr+inx*2*src2_data.u32+odd*16+2*size/8,size/8,&data3.s64); - mem->read(addr+inx*2*src2_data.u32+odd*16+3*size/8,size/8,&data4.s64); - mem->read(addr+inx*2*src2_data.u32+odd*16+4*size/8,size/8,&data5.s64); - mem->read(addr+inx*2*src2_data.u32+odd*16+5*size/8,size/8,&data6.s64); - mem->read(addr+inx*2*src2_data.u32+odd*16+6*size/8,size/8,&data7.s64); - mem->read(addr+inx*2*src2_data.u32+odd*16+7*size/8,size/8,&data8.s64); - printf("thread%d=%x,%x,%x,%x,%x,%x,%x,%x\n",0,data1.s64,data2.s64,data3.s64,data4.s64,data5.s64,data6.s64,data7.s64,data8.s64); - } - else if(type==F32_TYPE){ - mem->read(addr+inx*4*src2_data.u32+odd*32,size/8,&data1.s64); - mem->read(addr+inx*4*src2_data.u32+odd*32+size/8,size/8,&data2.s64); - mem->read(addr+inx*4*src2_data.u32+odd*32+2*size/8,size/8,&data3.s64); - mem->read(addr+inx*4*src2_data.u32+odd*32+3*size/8,size/8,&data4.s64); - mem->read(addr+inx*4*src2_data.u32+odd*32+4*size/8,size/8,&data5.s64); - mem->read(addr+inx*4*src2_data.u32+odd*32+5*size/8,size/8,&data6.s64); - mem->read(addr+inx*4*src2_data.u32+odd*32+6*size/8,size/8,&data7.s64); - mem->read(addr+inx*4*src2_data.u32+odd*32+7*size/8,size/8,&data8.s64); - printf("thread%d=%f,%f,%f,%f,%f,%f,%f,%f\n",thrd,data1.f32,data2.f32,data3.f32,data4.f32,data5.f32,data6.f32,data7.f32,data8.f32); + + addr_t new_addr = addr+thread_group_offset(thrd)*size/8; + mem->read(new_addr,size/8,&data1.s64); + mem->read(new_addr+2*size,size/8,&data2.s64); + mem->read(new_addr+4*size,size/8,&data3.s64); + mem->read(new_addr+6*size,size/8,&data4.s64); + mem->read(new_addr+8*size,size/8,&data5.s64); + mem->read(new_addr+10*size,size/8,&data6.s64); + mem->read(new_addr+12*size,size/8,&data7.s64); + mem->read(new_addr+14*size,size/8,&data8.s64); + + if(type==F16_TYPE) + printf("thread%d=%x,%x,%x,%x,%x,%x,%x,%x\n",thrd,data1.s64,data2.s64,data3.s64,data4.s64,data5.s64,data6.s64,data7.s64,data8.s64); + + else if(type==F32_TYPE) + printf("thread%d=%f,%f,%f,%f,%f,%f,%f,%f\n",thrd,data1.f32,data2.f32,data3.f32,data4.f32,data5.f32,data6.f32,data7.f32,data8.f32); + else + printf("wmma_ld:wrong type\n"); + + if(!((wmma_type==LOAD_C)&&(type==F16_TYPE))){ + thread->set_wmma_vector_operand_values(dst,data1,data2,data3,data4,data5,data6,data7,data8); } else{ - printf("wmma_ld:wrong type\n"); + ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4; + nw_data1.s64=((data1.s64 & 0xffff)<<16)|((data2.s64&0xffff)); + nw_data2.s64=((data3.s64 & 0xffff)<<16)|((data4.s64&0xffff)); + nw_data3.s64=((data5.s64 & 0xffff)<<16)|((data6.s64&0xffff)); + nw_data4.s64=((data7.s64 & 0xffff)<<16)|((data8.s64&0xffff)); + printf("wmma_load:data1.s64=%x,data2.s64=%x,new_data1.s64=%x\n",data1.s64,data2.s64,nw_data1.s64); + printf("wmma_load:data3.s64=%x,data4.s64=%x,new_data2.s64=%x\n",data3.s64,data4.s64,nw_data2.s64); + printf("wmma_load:data5.s64=%x,data6.s64=%x,new_data3.s64=%x\n",data5.s64,data6.s64,nw_data3.s64); + printf("wmma_load:data7.s64=%x,data8.s64=%x,new_data4.s64=%x\n",data7.s64,data8.s64,nw_data4.s64); + thread->set_vector_operand_values(dst,nw_data1,nw_data2,nw_data3,nw_data4); } - thread->set_wmma_vector_operand_values(dst,data1,data2,data3,data4,data5,data6,data7,data8); thread->m_last_effective_address = addr; thread->m_last_memory_space = space; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index 9a4d8d3..fb9adca 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -1083,6 +1083,27 @@ ptx_instruction::ptx_instruction( int opcode, int rr=0; std::list::const_iterator i; unsigned n=1; + for ( i=wmma_options.begin(); i!= wmma_options.end(); i++, n++ ) { + int last_ptx_inst_option = *i; + switch ( last_ptx_inst_option ) { + case SYNC_OPTION: + case LOAD_A: + case LOAD_B: + case LOAD_C: + case STORE_D: + case MMA: + m_wmma_type=last_ptx_inst_option; + break; + case ROW: + case COL: + case M16N16K16: + break; + default: + assert(0); + break; + } + } + n=1; for ( i=options.begin(); i!= options.end(); i++, n++ ) { int last_ptx_inst_option = *i; switch ( last_ptx_inst_option ) { diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 6bba717..7bc7522 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1025,6 +1025,9 @@ public: unsigned get_vector() const { return m_vector_spec;} unsigned get_atomic() const { return m_atomic_spec;} + int get_wmma_type() const { + return m_wmma_type; + } int get_type() const { assert( !m_scalar_type.empty() ); @@ -1134,9 +1137,9 @@ private: bool m_uni; //if branch instruction, this evaluates to true for uniform branches (ie jumps) bool m_to_option; unsigned m_cache_option; - unsigned m_wmma_type; - unsigned m_wmma_layout[2]; - unsigned m_wmma_configuration; + int m_wmma_type; + int m_wmma_layout[2]; + int m_wmma_configuration; unsigned m_rounding_mode; unsigned m_compare_op; unsigned m_saturation_mode; -- cgit v1.3 From a86a03769bbdd5ea99d194704086f9fbe82104c3 Mon Sep 17 00:00:00 2001 From: aamir Date: Mon, 11 Jun 2018 10:39:07 -0700 Subject: added all the configuration --- src/cuda-sim/instructions.cc | 543 ++++++++++++++++++++++++++++++------------- src/cuda-sim/ptx_ir.cc | 3 + src/cuda-sim/ptx_ir.h | 3 + 3 files changed, 382 insertions(+), 167 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 92c8529..e03cbce 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -56,43 +56,102 @@ const char *g_opcode_string[NUM_OPCODES] = { #undef OP_DEF #undef OP_W_DEF }; +//Using profiled information::check the TensorCoreMatrixArrangement.xls for details +unsigned thread_group_offset(int thread,unsigned wmma_type,unsigned wmma_layout,unsigned type){ -unsigned thread_group_offset(int thread){ + unsigned offset; + unsigned load_a_row[8]={0,128,0,128,64,192,64,192}; + unsigned load_a_col[8]={0,8,0,8,4,12,4,12}; + unsigned load_b_row[8]={0,8,0,8,4,12,4,12}; + unsigned load_b_col[8]={0,128,0,128,64,192,64,192}; + unsigned load_c_float_row[8]={0,128,8,136,64,192,72,200}; + unsigned load_c_float_col[8]={0,8,128,136,4,12,132,140}; + unsigned load_c_half_row[8]={0,128,8,136,64,192,72,200}; + unsigned load_c_half_col[8]={0,8,128,136,4,12,132,140}; unsigned thread_group=thread/4; unsigned in_tg_index=thread%4; - unsigned offset; - switch(thread_group){ - case 0: - offset=0; - break; - case 1: - offset=8; - break; - - case 2: - offset=128; - break; - case 3: - offset=136; - break; - case 4: - offset=4; - break; - case 5: - offset=12; - break; - case 6: - offset=132; + + switch(wmma_type){ + case LOAD_A: + if(wmma_layout==ROW) + offset=load_a_row[thread_group]+16*in_tg_index; + else + offset=load_a_col[thread_group]+16*in_tg_index; + break; + + + case LOAD_B: + if(wmma_layout==ROW) + offset=load_b_row[thread_group]+16*in_tg_index; + else + offset=load_b_col[thread_group]+16*in_tg_index; break; - case 7: - offset=140; + + case LOAD_C: + case STORE_D: + if(type==F16_TYPE){ + if(wmma_layout==ROW) + offset=load_c_half_row[thread_group]+16*in_tg_index; + else + offset=load_c_half_col[thread_group]+in_tg_index; + } + else{ + if(wmma_layout==ROW) + offset=load_c_float_row[thread_group]; + else + offset=load_c_float_col[thread_group]; + + switch(in_tg_index){ + case 0: + break; + case 1: + if(wmma_layout==ROW) + offset+=16; + else + offset+=1; + break; + case 2: + if(wmma_layout==ROW) + offset+=2; + else + offset+=32; + break; + case 3: + if(wmma_layout==ROW) + offset+=18; + else + offset+=33; + break; + default: + abort(); + } + } break; + default: abort(); } - return offset+in_tg_index; + + return offset; +} + +int acc_float_offset(int index,int wmma_layout){ + + int c_row_offset[]={0,1,32,33,4,5,36,37}; + int c_col_offset[]={0,16,2,18,64,80,66,82}; + + if(wmma_layout==ROW) + return c_row_offset[index]; + else if(wmma_layout==COL) + return c_col_offset[index]; + else{ + printf("wrong layout"); + abort(); + } + } + void inst_not_implemented( const ptx_instruction * pI ) ; ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread); @@ -1546,11 +1605,62 @@ unsigned trunc(unsigned num, unsigned precision) { } return num; } +void mapping(int thread,int wmma_type,int wmma_layout,int type,int index,int &row,int &col,int &assg_offset){ + int offset; + int c_row_offset[]={0,8,0,8,4,12,4,12}; + int c_col_offset[]={0,0,8,8,0,0,8,8}; + int c_tg_inside_row_offset[]={0,1,0,1}; + int c_tg_inside_col_offset[]={0,0,2,2}; + int c_inside_row_offset[]={0,0,2,2,0,0,2,2}; + int c_inside_col_offset[]={0,1,0,1,4,5,4,5}; + + offset=thread_group_offset(thread,wmma_type,wmma_layout,type); + + if(wmma_type==LOAD_A){ + if(wmma_layout==ROW){ + offset+=index+8*((thread%16)/8); + } + else{ + offset+=64*(index/4)+index%4+128*((thread%16)/8); + } + assg_offset=index+8*((thread%16)/8); + } + else if(wmma_type==LOAD_B){ + if(wmma_layout==ROW){ + offset+=64*(index/4)+index%4+128*((thread%16)/8); + } + else{ + offset+=index+8*((thread%16)/8); + } + assg_offset=index+8*((thread%16)/8); + } + else if( wmma_type==LOAD_C){ + if(type==F16_TYPE){ + row=c_row_offset[thread/4]+thread%4; + col=c_col_offset[thread/4]+index; + } + else{ + row=c_row_offset[thread/4]+c_tg_inside_row_offset[thread%4]+c_inside_row_offset[index]; + col=c_col_offset[thread/4]+c_tg_inside_col_offset[thread%4]+c_inside_col_offset[index]; + } + assg_offset=index; + } + if(wmma_type==LOAD_A||wmma_type==LOAD_B){ + if(wmma_layout==ROW){ + row=offset/16; + col=offset%16; + } + else{ + col=offset/16; + row=offset%16; + } + } +} void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) { int i,j,k,thrd; - int row,offset; + int row,col,offset; printf("mmaWorld\n"); ptx_reg_t matrix_a[16][16]; ptx_reg_t matrix_b[16][16]; @@ -1560,101 +1670,82 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) ptx_thread_info *thread; unsigned wmma_type = pI->get_wmma_type(); + unsigned a_layout = pI->get_wmma_layout(0); + unsigned b_layout = pI->get_wmma_layout(1); unsigned type = pI->get_type(); unsigned type2 = pI->get_type2(); int tid = inst.warp_id_func() * core->get_warp_size(); const operand_info &dst = pI->operand_lookup(0); -//NOT WOR thread = core->get_thread_info()[tid]; -//NOT WOR const operand_info &src_a= pI->operand_lookup(1); -//NOT WOR src_data= (thread->get_operand_value(src_a, dst, type, thread, 1)); -//NOT WOR thread->set_operand_value(dst, src_data, type, thread, pI); unsigned thread_group_index; + float temp; + half temp2; + for (thrd=0; thrd < core->get_warp_size(); thrd++){ thread = core->get_thread_info()[tid+thrd]; - printf("thread=%d:",thrd); + printf("THREAD=%d\n:",thrd); for(i=1;i<=3;i++){ - int k; const operand_info &src_a= pI->operand_lookup(i); unsigned nelem = src_a.get_vect_nelem(); ptx_reg_t v[8]; thread->get_vector_operand_values( src_a, v, nelem ); - if(i!=3){ - printf("%x ",v[0].f16); - printf("%x ",v[1].f16); - printf("%x ",v[2].f16); - printf("%x ",v[3].f16); - printf("%x ",v[4].f16); - printf("%x ",v[5].f16); - printf("%x ",v[6].f16); - printf("%x ",v[7].f16); - + + printf("Thread%d_Iteration=%d\n:",thrd,i); + for(k=0;k>16); + nw_v[k].f16 =*((half *)&hex_val); } - else{ - printf("%x ",v[0].s64); - printf("%x ",v[1].s64); - printf("%x ",v[2].s64); - printf("%x ",v[3].s64); + } + if(!((i==3)&&(type2==F32_TYPE))){ + for(k=0;k<2*nelem;k++){ + temp=nw_v[k].f16; + printf("%f ",temp); } + printf("\n"); + } + else{ + for(k=0;k<8;k++){ + printf("%f ",v[k].f32); + } + printf("\n"); } - thread_group_index=thread_group_offset(thrd); - row=(thread_group_index/16); - offset=thread_group_index%16; switch(i) { case 1 ://operand 1 - for(k=0;k<8;k++) - matrix_a[row+k][offset]=v[k]; - break; + for(k=0;k<8;k++){ + mapping(thrd,LOAD_A,a_layout,F16_TYPE,k,row,col,offset); + printf("A:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); + matrix_a[row][col]=nw_v[offset]; + } + break; case 2 ://operand 2 - for(k=0;k<8;k++) - matrix_b[row+k][offset]=v[k]; - break; + for(k=0;k<8;k++){ + mapping(thrd,LOAD_B,b_layout,F16_TYPE,k,row,col,offset); + printf("B:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); + matrix_b[row][col]=nw_v[offset]; + } + break; case 3 ://operand 3 - if(type2!=F16_TYPE){ - for(k=0;k<8;k++) - matrix_c[row+k][offset]=v[k]; - } - else { - ptx_reg_t nw_v[8]; - unsigned int n = 0x41933333; - float f = *((float*)&n); - int hex_val; - - for(k=0;k<8;k++){ - if(k%2==0) - hex_val=(v[k/2].s64&0xffff); - else - hex_val=((v[k/2].s64&0xffff0000)>>16); - nw_v[k].f16 =*((half *)&hex_val); - matrix_c[row+k][offset]=nw_v[k]; + for(k=0;k<8;k++){ + mapping(thrd,LOAD_C,ROW,type2,k,row,col,offset); + printf("C:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); + if(type2!=F16_TYPE){ + matrix_c[row][col]=v[offset]; + } + else { + matrix_c[row][col]=nw_v[offset]; } - printf("%x ",nw_v[0].f16); - printf("%x ",nw_v[1].f16); - printf("%x ",nw_v[2].f16); - printf("%x ",nw_v[3].f16); - printf("%x ",nw_v[4].f16); - printf("%x ",nw_v[5].f16); - printf("%x ",nw_v[6].f16); - printf("%x ",nw_v[7].f16); - //float t; - //int m; - //printf("\n"); - //for(m=0;m<8;m++){ - // t=nw_v[m].f16; - // printf(" %f ",t); - //} - //printf("\n"); } break; default : @@ -1667,22 +1758,26 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) printf("MATRIX_A\n"); for (i=0;i<16;i++){ for(j=0;j<16;j++){ - printf("%x ",matrix_a[i][j].f16); + temp=matrix_a[i][j].f16; + printf("%f ",temp); } printf("\n"); } printf("MATRIX_B\n"); for (i=0;i<16;i++){ for(j=0;j<16;j++){ - printf("%x ",matrix_b[i][j].f16); + temp=matrix_b[i][j].f16; + printf("%f ",temp); } printf("\n"); } printf("MATRIX_C\n"); for (i=0;i<16;i++){ for(j=0;j<16;j++){ - if(type2==F16_TYPE) - printf("%x ",matrix_c[i][j].f16); + if(type2==F16_TYPE){ + temp=matrix_c[i][j].f16; + printf("%f ",temp); + } else printf("%f ",matrix_c[i][j].f32); } @@ -1697,16 +1792,16 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) printf("MATRIX_D\n"); for (i=0;i<16;i++){ for(j=0;j<16;j++){ - if(type==F16_TYPE) - printf("%x ",matrix_d[i][j].f16); + if(type==F16_TYPE){ + temp=matrix_d[i][j].f16; + printf("%f ",temp); + } else printf("%.2f ",matrix_d[i][j].f32); } printf("\n"); } - float temp; - half temp2; for (i=0;i<16;i++){ for(j=0;j<16;j++){ for(k=0;k<16;k++){ @@ -1734,32 +1829,54 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) printf("MATRIX_D\n"); for (i=0;i<16;i++){ for(j=0;j<16;j++){ - if(type==F16_TYPE) - printf("%x ",matrix_d[i][j].f16); + if(type==F16_TYPE){ + temp=matrix_d[i][j].f16; + printf("%f ",temp); + } else printf("%.2f ",matrix_d[i][j].f32); } printf("\n"); } for (thrd=0; thrd < core->get_warp_size(); thrd++){ - thread_group_index=thread_group_offset(thrd); - row=(thread_group_index/16); - offset=thread_group_index%16; + int row_t[8]; + int col_t[8]; + for(k=0;k<8;k++){ + mapping(thrd,LOAD_C,ROW,type,k,row_t[k],col_t[k],offset); + printf("mma:store:row:%d,col%d\n",row_t[k],col_t[k]); + } thread = core->get_thread_info()[tid+thrd]; - //r2=dst.get_symbol(); - //printf("thrd=%d,i=%d,register%s, data=%f\n",thrd,i,(r2->name()).c_str(),matrix_d[row][offset+i].f32); - //thread->set_operand_value(dst, matrix_d[row][offset+i], type, thread, pI); + + if(type==F32_TYPE){ - thread->set_wmma_vector_operand_values(dst,matrix_d[row][offset],matrix_d[row+1][offset],matrix_d[row+2][offset],matrix_d[row+3][offset],matrix_d[row+4][offset],matrix_d[row+5][offset],matrix_d[row+6][offset],matrix_d[row+7][offset]); - printf("thread%d=%x,%x,%x,%x",thrd,matrix_d[row][offset].f16,matrix_d[row+1][offset].f16,matrix_d[row+2][offset].f16,matrix_d[row+3][offset].f16); - printf(",%x,%x,%x,%x\n",matrix_d[row+4][offset].f16,matrix_d[row+5][offset].f16,matrix_d[row+6][offset].f16,matrix_d[row+7][offset].f16); - } + thread->set_wmma_vector_operand_values(dst,matrix_d[row_t[0]][col_t[0]],matrix_d[row_t[1]][col_t[1]],matrix_d[row_t[2]][col_t[2]],matrix_d[row_t[3]][col_t[3]],matrix_d[row_t[4]][col_t[4]],matrix_d[row_t[5]][col_t[5]],matrix_d[row_t[6]][col_t[6]],matrix_d[row_t[7]][col_t[7]]); + + printf("thread%d:",thrd); + for(k=0;k<8;k++){ + printf("%f ",matrix_d[row_t[k]][col_t[k]].f32); + } + printf("\n"); + } else if(type==F16_TYPE){ + + printf("thread%d:",thrd); + for(k=0;k<8;k++){ + temp=matrix_d[row_t[k]][col_t[k]].f16; + printf("%f ",temp); + } + printf("\n"); + + printf("thread%d:",thrd); + for(k=0;k<8;k++){ + printf("%x ",matrix_d[row_t[k]][col_t[k]].f16); + } + printf("\n"); + ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4; - nw_data1.s64=((matrix_d[row][offset].s64 & 0xffff))|((matrix_d[row+1][offset].s64&0xffff)<<16); - nw_data2.s64=((matrix_d[row+2][offset].s64 & 0xffff))|((matrix_d[row+3][offset].s64&0xffff)<<16); - nw_data3.s64=((matrix_d[row+4][offset].s64 & 0xffff))|((matrix_d[row+5][offset].s64&0xffff)<<16); - nw_data4.s64=((matrix_d[row+6][offset].s64 & 0xffff))|((matrix_d[row+7][offset].s64&0xffff)<<16); + nw_data1.s64=((matrix_d[row_t[0]][col_t[0]].s64 & 0xffff))|((matrix_d[row_t[1]][col_t[1]].s64&0xffff)<<16); + nw_data2.s64=((matrix_d[row_t[2]][col_t[2]].s64 & 0xffff))|((matrix_d[row_t[3]][col_t[3]].s64&0xffff)<<16); + nw_data3.s64=((matrix_d[row_t[4]][col_t[4]].s64 & 0xffff))|((matrix_d[row_t[5]][col_t[5]].s64&0xffff)<<16); + nw_data4.s64=((matrix_d[row_t[6]][col_t[6]].s64 & 0xffff))|((matrix_d[row_t[7]][col_t[7]].s64&0xffff)<<16); thread->set_vector_operand_values(dst,nw_data1,nw_data2,nw_data3,nw_data4); printf("thread%d=%x,%x,%x,%x",thrd,nw_data1.s64,nw_data2.s64,nw_data3.s64,nw_data4.s64); @@ -2791,9 +2908,10 @@ void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) int tid = inst.warp_id_func()*core->get_warp_size(); unsigned type = pI->get_type(); unsigned wmma_type = pI->get_wmma_type(); + unsigned wmma_layout = pI->get_wmma_layout(0); for (thrd=0; thrd < core->get_warp_size(); thrd++) { - thread = core->get_thread_info()[tid+thrd]; + thread = core->get_thread_info()[tid+thrd]; odd=thrd%2; inx=thrd/2; ptx_reg_t addr_reg = thread->get_operand_value(src1, src, type, thread, 1); @@ -2812,7 +2930,7 @@ void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) type_info_key::type_decode(type,size,t); printf("mma_st: thrd=%d,addr=%d, fp(size=%d), stride=%d\n",thrd,addr_reg.u32,size,src2_data.u32); - addr_t new_addr = addr+thread_group_offset(thrd)*size/8; + addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type)*size/8; ptx_reg_t nw_v[8]; for(k=0;k<8;k++){ @@ -2824,11 +2942,24 @@ void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) for(k=0;k<8;k++){ if(type==F32_TYPE){ - mem->write(new_addr+k*2*size,size/8,&v[k].s64,thread,pI); + mem->write(new_addr+4*acc_float_offset(k,wmma_layout),size/8,&v[k].s64,thread,pI); + printf("wmma:store:thread%d=%x,%x,%x,%x,%x,%x,%x,%x\n",thrd,v[0].s64,v[1].s64,v[2].s64,v[3].s64,v[4].s64,v[5].s64,v[6].s64,v[7].s64); + float temp; + int l; + printf("thread=%d:",thrd); + for(l=0;l<8;l++){ + temp=v[0].f32; + printf("%f",temp); + } + printf("\n"); + } else if(type==F16_TYPE){ - mem->write(new_addr+k*2*size,size/8,&nw_v[k].s64,thread,pI); + if(wmma_layout==ROW) + mem->write(new_addr+k*2,size/8,&nw_v[k].s64,thread,pI); + else if(wmma_layout==COL) + mem->write(new_addr+k*32,size/8,&nw_v[k].s64,thread,pI); printf("wmma:store:thread%d=%x,%x,%x,%x,%x,%x,%x,%x\n",thrd,nw_v[0].s64,nw_v[1].s64,nw_v[2].s64,nw_v[3].s64,nw_v[4].s64,nw_v[5].s64,nw_v[6].s64,nw_v[7].s64); } } @@ -2838,24 +2969,24 @@ void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) thread->m_last_memory_space = space; } } + void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) { size_t size; - int t; + int t,i; const operand_info &dst = pI->dst(); const operand_info &src1 = pI->src1(); const operand_info &src2 = pI->src2(); unsigned type = pI->get_type(); unsigned wmma_type = pI->get_wmma_type(); - + unsigned wmma_layout = pI->get_wmma_layout(0); int tid = inst.warp_id_func()*core->get_warp_size(); - int thrd,odd,inx; + int thrd; ptx_thread_info *thread; + for (thrd=0; thrd < core->get_warp_size(); thrd++){ thread = core->get_thread_info()[tid+thrd]; - odd=thrd%2; - inx=thrd/2; ptx_reg_t src1_data = thread->get_operand_value(src1, dst, U32_TYPE, thread, 1); ptx_reg_t src2_data = thread->get_operand_value(src2, dst, U32_TYPE, thread, 1); @@ -2863,46 +2994,124 @@ void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) memory_space *mem = NULL; addr_t addr = src1_data.u32; - decode_space(space,thread,src1,mem,addr); - type_info_key::type_decode(type,size,t); - ptx_reg_t data1, data2, data3, data4; - ptx_reg_t data5, data6, data7, data8; - printf("mma_ld: thrd=%d,addr=%d, fp16(size=%d), stride=%d\n",thrd,src1_data.u32,size,src2_data.u32); - addr_t new_addr = addr+thread_group_offset(thrd)*size/8; - mem->read(new_addr,size/8,&data1.s64); - mem->read(new_addr+2*size,size/8,&data2.s64); - mem->read(new_addr+4*size,size/8,&data3.s64); - mem->read(new_addr+6*size,size/8,&data4.s64); - mem->read(new_addr+8*size,size/8,&data5.s64); - mem->read(new_addr+10*size,size/8,&data6.s64); - mem->read(new_addr+12*size,size/8,&data7.s64); - mem->read(new_addr+14*size,size/8,&data8.s64); - - if(type==F16_TYPE) - printf("thread%d=%x,%x,%x,%x,%x,%x,%x,%x\n",thrd,data1.s64,data2.s64,data3.s64,data4.s64,data5.s64,data6.s64,data7.s64,data8.s64); + ptx_reg_t data[16]; + printf("mma_ld: thrd=%d,addr=%d, fpsize=%d, stride=%d\n",thrd,src1_data.u32,size,src2_data.u32); + addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type)*size/8; + + if(wmma_type==LOAD_A){ + for(i=0;i<16;i++){ + if(wmma_layout==ROW) + mem->read(new_addr+2*i,size/8,&data[i].s64); + else if(wmma_layout==COL){ + mem->read(new_addr+2*(i%4)+128*(i/4),size/8,&data[i].s64); + } + else{ + printf("mma_ld:wrong_layout_type\n"); + abort(); + } + } + } + else if(wmma_type==LOAD_B){ + for(i=0;i<16;i++){ + if(wmma_layout==COL) + mem->read(new_addr+2*i,size/8,&data[i].s64); + else if(wmma_layout==ROW){ + mem->read(new_addr+2*(i%4)+128*(i/4),size/8,&data[i].s64); + } + else{ + printf("mma_ld:wrong_layout_type\n"); + abort(); + } + } + } + else if(wmma_type==LOAD_C){ + for(i=0;i<8;i++){ + if(type==F16_TYPE){ + if(wmma_layout==ROW) + mem->read(new_addr+2*i,size/8,&data[i].s64); + else if(wmma_layout==COL) + mem->read(new_addr+32*i,size/8,&data[i].s64); + else{ + printf("mma_ld:wrong_type\n"); + abort(); + } + } + else if(type==F32_TYPE){ + mem->read(new_addr+4*acc_float_offset(i,wmma_layout),size/8,&data[i].s64); + } + else{ + printf("wrong type"); + abort(); + } + } + } + else{ + printf("wrong wmma type\n");; + abort(); + } - else if(type==F32_TYPE) - printf("thread%d=%f,%f,%f,%f,%f,%f,%f,%f\n",thrd,data1.f32,data2.f32,data3.f32,data4.f32,data5.f32,data6.f32,data7.f32,data8.f32); - else - printf("wmma_ld:wrong type\n"); + if(type==F16_TYPE){ + printf("\nthread%d= ",thrd); + for(i=0;i<16;i++){ + printf("%x ",data[i].u64); + } + printf("\n"); + + printf("\nthread%d= ",thrd); + float temp; + for(i=0;i<16;i++){ + temp=data[i].f16; + printf("%f ",temp); + } + printf("\n"); + } + else{ + printf("\nthread%d= ",thrd); + for(i=0;i<8;i++){ + printf("%f ",data[i].f32); + } + printf("\n"); + printf("\nthread%d= ",thrd); + for(i=0;i<8;i++){ + printf("%x ",data[i].u64); + } + printf("\n"); + } - if(!((wmma_type==LOAD_C)&&(type==F16_TYPE))){ - thread->set_wmma_vector_operand_values(dst,data1,data2,data3,data4,data5,data6,data7,data8); + if((wmma_type==LOAD_C)&&(type==F32_TYPE)){ + thread->set_wmma_vector_operand_values(dst,data[0],data[1],data[2],data[3],data[4],data[5],data[6],data[7]); } else{ - ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4; - nw_data1.s64=((data1.s64 & 0xffff))|((data2.s64&0xffff)<<16); - nw_data2.s64=((data3.s64 & 0xffff))|((data4.s64&0xffff)<<16); - nw_data3.s64=((data5.s64 & 0xffff))|((data6.s64&0xffff)<<16); - nw_data4.s64=((data7.s64 & 0xffff))|((data8.s64&0xffff)<<16); - printf("wmma_load:data1.s64=%x,data2.s64=%x,new_data1.s64=%x\n",data1.s64,data2.s64,nw_data1.s64); - printf("wmma_load:data3.s64=%x,data4.s64=%x,new_data2.s64=%x\n",data3.s64,data4.s64,nw_data2.s64); - printf("wmma_load:data5.s64=%x,data6.s64=%x,new_data3.s64=%x\n",data5.s64,data6.s64,nw_data3.s64); - printf("wmma_load:data7.s64=%x,data8.s64=%x,new_data4.s64=%x\n",data7.s64,data8.s64,nw_data4.s64); - thread->set_vector_operand_values(dst,nw_data1,nw_data2,nw_data3,nw_data4); + ptx_reg_t nw_data[8]; + int num_reg; + + if(wmma_type==LOAD_C) + num_reg=4; + else + num_reg=8; + + for(i=0;iset_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3]); + else + thread->set_wmma_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3],nw_data[4],nw_data[5],nw_data[6],nw_data[7]); + + printf("wmma_load:data[0].s64=%x,data[1].s64=%x,new_data[0].s64=%x\n",data[0].u64,data[1].u64,nw_data[0].u64); + printf("wmma_load:data[2].s64=%x,data[3].s64=%x,new_data[1].s64=%x\n",data[2].u64,data[3].u64,nw_data[1].u64); + printf("wmma_load:data[4].s64=%x,data[5].s64=%x,new_data[2].s64=%x\n",data[4].u64,data[5].u64,nw_data[2].u64); + printf("wmma_load:data[6].s64=%x,data[7].s64=%x,new_data[3].s64=%x\n",data[6].u64,data[7].u64,nw_data[3].u64); + if(wmma_type!=LOAD_C){ + printf("wmma_load:data[8].s64=%x,data[9].s64=%x,new_data[4].s64=%x\n",data[8].u64,data[9].u64,nw_data[4].s64); + printf("wmma_load:data[10].s64=%x,data[11].s64=%x,new_data[5].s64=%x\n",data[10].u64,data[11].u64,nw_data[5].u64); + printf("wmma_load:data[12].s64=%x,data[13].s64=%x,new_data[6].s64=%x\n",data[12].u64,data[13].u64,nw_data[6].u64); + printf("wmma_load:data[14].s64=%x,data[15].s64=%x,new_data[7].s64=%x\n",data[14].u64,data[15].u64,nw_data[3].u64); + } } thread->m_last_effective_address = addr; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index fb9adca..9c2ac69 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -1096,6 +1096,8 @@ ptx_instruction::ptx_instruction( int opcode, break; case ROW: case COL: + m_wmma_layout[rr++]=last_ptx_inst_option; + break; case M16N16K16: break; default: @@ -1103,6 +1105,7 @@ ptx_instruction::ptx_instruction( int opcode, break; } } + rr=0; n=1; for ( i=options.begin(); i!= options.end(); i++, n++ ) { int last_ptx_inst_option = *i; diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 7bc7522..62d7c7c 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1028,6 +1028,9 @@ public: int get_wmma_type() const { return m_wmma_type; } + int get_wmma_layout(int index) const { + return m_wmma_layout[index];//0->Matrix D,1->Matrix C + } int get_type() const { assert( !m_scalar_type.empty() ); -- cgit v1.3 From 37dc5311a75548b848a33a3b7369ce4bee64b444 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Thu, 14 Jun 2018 20:15:52 -0700 Subject: bfe bug fix --- .gitignore | 5 +++++ src/cuda-sim/instructions.cc | 30 +++++++++++++++--------------- src/cuda-sim/ptx_ir.h | 2 +- 3 files changed, 21 insertions(+), 16 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/.gitignore b/.gitignore index 0e2a898..e4e1631 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,8 @@ cscope* tags regression.sh +#gcov +*.gcov +*.gcda +*.gcno + diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 08bf528..c77e4da 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -1353,45 +1353,45 @@ void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread ) const operand_info &src1 = pI->src1(); const operand_info &src2 = pI->src2(); const operand_info &src3 = pI->src3(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t src = thread->get_operand_value(src1, dst, i_type, thread, 1); ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + ptx_reg_t data; unsigned pos = b.u32 & 0xFF; unsigned len = c.u32 & 0xFF; - unsigned d = 0; switch (i_type) { case U32_TYPE: { unsigned mask; - d = a.u32 >> pos; + data.u32 = src.u32 >> pos; mask = 0xFFFFFFFF >> (32 - len); - d &= mask; + data.u32 &= mask; break; } case U64_TYPE: { unsigned long mask; - d = a.u64 >> pos; + data.u64 = src.u64 >> pos; mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - d &= mask; + data.u64 &= mask; break; } case S32_TYPE: { unsigned mask; unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (a.s32 >> min) & 0x1; - d = a.s32 >> pos; + unsigned sbit = len == 0 ? 0 : (src.s32 >> min) & 0x1; + data.s32 = src.s32 >> pos; if (sbit > 0) { mask = 0xFFFFFFFF << len; - d |= mask; + data.s32 |= mask; } else { mask = 0xFFFFFFFF >> (32 - len); - d &= mask; + data.s32 &= mask; } break; } @@ -1399,17 +1399,17 @@ void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { unsigned long mask; unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (a.s64 >> min) & 0x1; - d = a.s64 >> pos; + unsigned sbit = len == 0 ? 0 : (src.s64 >> min) & 0x1; + data.s64 = src.s64 >> pos; if (sbit > 0) { mask = 0xFFFFFFFFFFFFFFFF << len; - d |= mask; + data.s64 |= mask; } else { mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - d &= mask; + data.s64 &= mask; } break; } @@ -1418,7 +1418,7 @@ void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread ) abort(); return; } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } void bfi_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 58d5f49..5b68fcf 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -745,7 +745,7 @@ public: { ptx_reg_t result; switch ( m_type ) { - case int_t: result.s32 = m_value.m_int; break; + case int_t: result.s64 = m_value.m_int; break; case float_op_t: result.f32 = m_value.m_float; break; case double_op_t: result.f64 = m_value.m_double; break; case unsigned_t: result.u32 = m_value.m_unsigned; break; -- cgit v1.3 From 76a124e9186b9574858238d423b9c5ce715f2c32 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Thu, 21 Jun 2018 17:35:07 -0700 Subject: WIP adding support for PTX JIT and dumping params to cudaLaunches --- libcuda/cuda_runtime_api.cc | 168 ++++++++++++++++++++++++++++++++++++++++--- src/cuda-sim/cuda-sim.cc | 81 +++++++++++++++++++++ src/cuda-sim/cuda-sim.h | 1 + src/cuda-sim/instructions.cc | 1 + src/cuda-sim/ptx_ir.cc | 8 +++ src/cuda-sim/ptx_ir.h | 2 + 6 files changed, 252 insertions(+), 9 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index 08ee413..300fa28 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -126,6 +126,9 @@ #include "host_defines.h" #include "builtin_types.h" #include "driver_types.h" +#if (CUDART_VERSION >= 8000) +#include "cuda.h" +#endif #if (CUDART_VERSION < 8000) #include "__cudaFatFormat.h" #endif @@ -287,6 +290,34 @@ struct CUctx_st { } } + void register_hostFun_function( const char*hostFun, function_info* f){ + m_kernel_lookup[hostFun] = f; + } + + dim3 get_blockdim(const char *hostFun) + { + std::map::iterator i=m_hostFun_blockdim.find(hostFun); + assert( i != m_hostFun_blockdim.end() ); + return i->second; + } + + dim3 get_griddim(const char *hostFun) + { + std::map::iterator i=m_hostFun_griddim.find(hostFun); + assert( i != m_hostFun_griddim.end() ); + return i->second; + } + + void set_blockdim(const char *hostFun, dim3 dims) + { + m_hostFun_blockdim[hostFun] = dims; + } + + void set_griddim(const char *hostFun, dim3 dims) + { + m_hostFun_griddim[hostFun] = dims; + } + function_info *get_kernel(const char *hostFun) { std::map::iterator i=m_kernel_lookup.find(hostFun); @@ -298,6 +329,8 @@ private: _cuda_device_id *m_gpu; // selected gpu std::map m_code; // fat binary handle => global symbol table unsigned m_last_fat_cubin_handle; + std::map m_hostFun_blockdim; + std::map m_hostFun_griddim; std::map m_kernel_lookup; // unique id (CUDA app function address) => kernel entry point struct gpgpu_ptx_sim_info m_binary_info; @@ -1300,16 +1333,15 @@ int CUDARTAPI __cudaSynchronizeThreads(void**, void*) * * *******************************************************************************/ -#if (CUDART_VERSION >= 3010) +#if (CUDART_VERSION >= 3010 && CUDART_VERSION < 8000) typedef struct CUuuid_st { /**< CUDA definition of UUID */ char bytes[16]; } CUuuid; -/** - * CUDA UUID types - */ -// typedef __device_builtin__ struct CUuuid_st cudaUUID_t; +#endif + +#if (CUDART_VERSION >= 3010) __host__ cudaError_t CUDARTAPI cudaGetExportTable(const void **ppExportTable, const cudaUUID_t *pExportTableId) { @@ -1958,10 +1990,10 @@ void cuobjdumpParseBinary(unsigned int handle){ symtab = gpgpu_ptx_sim_load_ptx_from_filename( ptx_filename.c_str() ); } } - name_symtab[fname] = symtab; - context->add_binary(symtab, handle); - load_static_globals(symtab,STATIC_ALLOC_LIMIT,0xFFFFFFFF,context->get_device()->get_gpgpu()); - load_constants(symtab,STATIC_ALLOC_LIMIT,context->get_device()->get_gpgpu()); + name_symtab[fname] = symtab; + context->add_binary(symtab, handle); + load_static_globals(symtab,STATIC_ALLOC_LIMIT,0xFFFFFFFF,context->get_device()->get_gpgpu()); + load_constants(symtab,STATIC_ALLOC_LIMIT,context->get_device()->get_gpgpu()); return; #endif @@ -2602,5 +2634,123 @@ kernel_info_t *gpgpu_cuda_ptx_sim_init_grid( const char *hostFun, g_ptx_kernel_count++; fflush(stdout); + if(g_debug_execution >= 3){ + entry->debug_param(); + } + return result; } + +CUresult CUDAAPI cuLinkCreate(unsigned int numOptions, CUjit_option *options, void **optionValues, CUlinkState *stateOut) +{ + //currently do not support options or multiple CUlinkStates + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuLinkAddData(CUlinkState state, CUjitInputType type, void *data, size_t size, const char *name, + unsigned int numOptions, CUjit_option *options, void **optionValues) +{ + assert(type==CU_JIT_INPUT_PTX); + cuda_not_implemented(__my_func__,__LINE__); + return CUDA_ERROR_UNKNOWN; +} + +CUresult CUDAAPI cuLinkAddFile(CUlinkState state, CUjitInputType type, const char *path, + unsigned int numOptions, CUjit_option *options, void **optionValues) +{ + static bool addedFile = false; + if (addedFile){ + printf("GPGPU-Sim PTX: ERROR: cuLinkAddFile does not support multiple file"); + abort(); + } + + //blocking + assert(type==CU_JIT_INPUT_PTX); + CUctx_st *context = GPGPUSim_Context(); + char *file = getenv("PTX_JIT_PATH"); + if(file==NULL){ + printf("GPGPU-Sim PTX: ERROR: PTX_JIT_PATH has not been set"); + abort(); + } + strcat(file,path); + symbol_table *symtab = gpgpu_ptx_sim_load_ptx_from_filename( file ); + std::string fname(path); + name_symtab[fname] = symtab; + context->add_binary(symtab, 1); + load_static_globals(symtab,STATIC_ALLOC_LIMIT,0xFFFFFFFF,context->get_device()->get_gpgpu()); + load_constants(symtab,STATIC_ALLOC_LIMIT,context->get_device()->get_gpgpu()); + addedFile = true; + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuLinkComplete(CUlinkState state, void **cubinOut, size_t *sizeOut) +{ + //all cuLink* function are implemented to block until completion so nothing to do here + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuLinkDestroy(CUlinkState state) +{ + //currently do not support options or multiple CUlinkStates + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuModuleLoadData(CUmodule *module, const void *image) +{ + //Currently do not support multiple modules + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuModuleGetFunction(CUfunction *hfunc, CUmodule hmod, const char *name) +{ + CUctx_st* context = GPGPUSim_Context(); + std::string key(name); + //only support one file + assert(name_symtab.size()==1); + symbol_table* symtab = name_symtab.begin()->second; + function_info* f = symtab->lookup_function( std::string(name) ); + //just need to add given pointer to map for cudaLaunch + context->register_hostFun_function( (const char*) hfunc, f); + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuModuleUnload(CUmodule hmod) +{ + //Currently do not support multiple modules + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuFuncSetBlockShape(CUfunction hfunc, int x, int y, int z) +{ + CUctx_st* context = GPGPUSim_Context(); + dim3 dims(x,y,z); + context->set_blockdim((const char *)hfunc, dims); + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuParamSetSize(CUfunction hfunc, unsigned int numbytes) +{ + //Nothing to do + return CUDA_SUCCESS; +} + +CUresult CUDAAPI cuParamSetv(CUfunction hfunc, int offset, void *ptr, unsigned int numbytes) +{ + cuda_not_implemented(__my_func__,__LINE__); + return CUDA_ERROR_UNKNOWN; +} + +CUresult CUDAAPI cuLaunchGrid(CUfunction f, int grid_width, int grid_height) +{ + cuda_not_implemented(__my_func__,__LINE__); + return CUDA_ERROR_UNKNOWN; + + CUctx_st* context = GPGPUSim_Context(); + const char *hostFun = (const char*) f; + dim3 dims(grid_width,grid_height,1); + context->set_griddim((const char *)f, dims); + cudaConfigureCall(context->get_griddim(hostFun), context->get_blockdim(hostFun), 0, NULL); + + cudaLaunch(hostFun); + return CUDA_SUCCESS; +} diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 34368ce..6875edd 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1226,6 +1226,87 @@ void function_info::list_param( FILE *fout ) const fflush(fout); } +void function_info::debug_param( ) const +{ + char filename[] = "params.txt"; + char buff[1024]; + snprintf(buff,1024,"c++filt %s > %s", get_name().c_str(), filename); + system(buff); + FILE *fp = fopen(filename, "r"); + fgets(buff, 1024, fp); + fclose(fp); + + std::string fn(buff); + size_t pos1, pos2; + pos1 = fn.find("("); + pos2 = fn.find(")"); + assert(pos2>pos1&&pos1>0); + strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); + printf("params: %s\n", buff); + char *tok; + std::vector params; + tok = strtok(buff, ","); + while(tok!=NULL){ + std::string param(tok); + param.erase(0, param.find_first_not_of(" ")); + param.erase(param.find_last_not_of(" ")+1); + params.push_back(param); + tok = strtok(NULL, ","); + } + for (auto const& it : params){ + std::cout<::const_iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + const param_info &p = i->second; + std::string name = p.get_name(); + param_t param_value = p.get_value(); + if(params[i->first].find("const")!=std::string::npos){ + fprintf(fout, "Input: "); + } else { + fprintf(fout, "Input/output: "); + } + + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); + fprintf(fout, "%s: %#08x, ", name.c_str(), param_addr); + + if(params[i->first].find("int")!=std::string::npos){ + size_t len = param_value.size/sizeof(int); + int val[len]; + memcpy((void*) val, param_value.pdata+param_value.offset, param_value.size); + fprintf(fout, "val (int) = "); + for (unsigned i = 0; ifirst].find("float")!=std::string::npos){ + size_t len = param_value.size/sizeof(float); + float val[len]; + memcpy((void*) val, param_value.pdata+param_value.offset, param_value.size); + fprintf(fout, "val (float) = "); + for (unsigned i = 0; i bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc) { diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h index 958daba..9049a84 100644 --- a/src/cuda-sim/cuda-sim.h +++ b/src/cuda-sim/cuda-sim.h @@ -32,6 +32,7 @@ #include"../gpgpu-sim/shader.h" #include #include +#include #include #include"ptx_sim.h" diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index c77e4da..034a7b9 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -2753,6 +2753,7 @@ void mov_impl( const ptx_instruction *pI, ptx_thread_info *thread ) const operand_info &dst = pI->dst(); const operand_info &src1 = pI->src1(); unsigned i_type = pI->get_type(); + assert( src1.is_param_local() == 0 ); if( (src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && (i_type != BB128_TYPE) && (i_type != FF64_TYPE) ) { // pack or unpack operation diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index 016c600..482b9e0 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -257,6 +257,14 @@ bool symbol_table::add_function_decl( const char *name, int entry_point, functio return prior_decl; } +function_info *symbol_table::lookup_function( std::string name ) +{ + std::string key = std::string(name); + std::map::iterator it = m_function_info_lookup.find(key); + assert ( it != m_function_info_lookup.end() ); + return it->second; +} + type_info *symbol_table::add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ) { if( space_spec == param_space_unclassified ) diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 5b68fcf..341f9b7 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -313,6 +313,7 @@ public: symbol *add_variable( const char *identifier, const type_info *type, unsigned size, const char *filename, unsigned line ); void add_function( function_info *func, const char *filename, unsigned linenumber ); bool add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **symbol_table ); + function_info *lookup_function(std::string name); type_info *add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ); type_info *add_type( function_info *func ); type_info *get_array_type( type_info *base_type, unsigned array_dim ); @@ -1256,6 +1257,7 @@ public: void finalize( memory_space *param_mem ); void param_to_shared( memory_space *shared_mem, symbol_table *symtab ); void list_param( FILE *fout ) const; + void debug_param() const; const struct gpgpu_ptx_sim_info* get_kernel_info () const { -- cgit v1.3 From 1f77b0720f69d684db83beb7a5513cd9461e3676 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Wed, 27 Jun 2018 15:26:00 -0700 Subject: WIP dump params --- debug_tools/WatchYourStep/ptxjitplus/ptxjitplus | Bin 139317 -> 0 bytes .../WatchYourStep/ptxjitplus/ptxjitplus.cpp | 112 ++++++++++++++++++--- libcuda/cuda_runtime_api.cc | 12 ++- src/cuda-sim/cuda-sim.cc | 59 +++++++++-- src/cuda-sim/ptx_ir.h | 2 +- 5 files changed, 157 insertions(+), 28 deletions(-) delete mode 100755 debug_tools/WatchYourStep/ptxjitplus/ptxjitplus (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus b/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus deleted file mode 100755 index ddc3435..0000000 Binary files a/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus and /dev/null differ diff --git a/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp b/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp index 9954e31..8645114 100644 --- a/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp +++ b/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp @@ -111,6 +111,87 @@ void ptxJIT(int argc, char **argv, CUmodule *phModule, CUfunction *phKernel, CUl checkCudaErrors(cuLinkDestroy(*lState)); } +void function_info::debug_param( ) const +{ + char filename[] = "params.txt"; + char buff[1024]; + snprintf(buff,1024,"c++filt %s > %s", get_name().c_str(), filename); + system(buff); + FILE *fp = fopen(filename, "r"); + fgets(buff, 1024, fp); + fclose(fp); + + std::string fn(buff); + size_t pos1, pos2; + pos1 = fn.find("("); + pos2 = fn.find(")"); + assert(pos2>pos1&&pos1>0); + strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); + printf("params: %s\n", buff); + char *tok; + std::vector params; + tok = strtok(buff, ","); + while(tok!=NULL){ + std::string param(tok); + param.erase(0, param.find_first_not_of(" ")); + param.erase(param.find_last_not_of(" ")+1); + params.push_back(param); + tok = strtok(NULL, ","); + } + for (auto const& it : params){ + std::cout<::const_iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + const param_info &p = i->second; + std::string name = p.get_name(); + param_t param_value = p.get_value(); + if(params[i->first].find("const")!=std::string::npos){ + fprintf(fout, "Input: "); + } else { + fprintf(fout, "Input/output: "); + } + + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); + fprintf(fout, "%s: %#08x, ", name.c_str(), param_addr); + + if(params[i->first].find("int")!=std::string::npos){ + size_t len = param_value.size/sizeof(int); + int val[len]; + memcpy((void*) val, param_value.pdata+param_value.offset, param_value.size); + fprintf(fout, "val (int) = "); + for (unsigned i = 0; ifirst].find("float")!=std::string::npos){ + size_t len = param_value.size/sizeof(float); + float val[len]; + memcpy((void*) val, param_value.pdata+param_value.offset, param_value.size); + fprintf(fout, "val (float) = "); + for (unsigned i = 0; i >& param_data) { char *wys_exec_path = getenv("WYS_EXEC_PATH"); @@ -160,19 +241,17 @@ int main(int argc, char **argv) { const unsigned int nThreads = 256; const unsigned int nBlocks = 64; - const size_t memSize = nThreads * nBlocks * sizeof(int); CUmodule hModule = 0; CUfunction hKernel = 0; CUlinkState lState; - int *d_data = 0; - int *h_data = 0; int cuda_device = 0; cudaDeviceProp deviceProp; printf("[%s] - Starting...\n", sSDKname); std::vector< std::pair > param_data; + std::vector< std::pair > device_data; void* storedReg = initializeData(param_data); if (checkCmdLineFlag(argc, (const char **)argv, "device")) @@ -213,15 +292,7 @@ int main(int argc, char **argv) exit(EXIT_WAIVED); } - // Allocate memory on host and device (Runtime API) - // NOTE: The runtime API will create the GPU Context implicitly here - if ((h_data = (int *)malloc(memSize)) == NULL) - { - std::cerr << "Could not allocate host memory" << std::endl; - exit(EXIT_FAILURE); - } - checkCudaErrors(cudaMalloc(&d_data, memSize)); // JIT Compile the Kernel from PTX and get the Handles (Driver API) ptxJIT(argc, argv, &hModule, &hKernel, &lState); @@ -229,16 +300,29 @@ int main(int argc, char **argv) // Set the kernel parameters (Driver API) checkCudaErrors(cuFuncSetBlockShape(hKernel, nThreads, 1, 1)); - //param_data + + //Initialize param_data for kernel int paramOffset = 0; - checkCudaErrors(cuParamSetv(hKernel, paramOffset, &d_data, sizeof(d_data))); - paramOffset += sizeof(d_data); + for( std::vector< std::pair >::const_iterator i=param_data.begin(); i!=param_data.end(); i++ ) { + size_t memSize = nThreads * nBlocks * i->first; + unsigned char *d_data = 0; + checkCudaErrors(cudaMalloc((void**)&d_data, memSize)); + checkCudaErrors(cudaMemcpy(d_data,i->first,cudaMemcpyHostToDevice)); + checkCudaErrors(cuParamSetv(hKernel, paramOffset, &d_data, memSize)); + paramOffset += i->first; + } checkCudaErrors(cuParamSetSize(hKernel, paramOffset)); // Launch the kernel (Driver API_) checkCudaErrors(cuLaunchGrid(hKernel, nBlocks, 1)); std::cout << "CUDA kernel launched" << std::endl; + int *h_data = 0; + if ((h_data = (int *)malloc(memSize)) == NULL) + { + std::cerr << "Could not allocate host memory" << std::endl; + exit(EXIT_FAILURE); + } // Copy the result back to the host checkCudaErrors(cudaMemcpy(h_data, d_data, memSize, cudaMemcpyDeviceToHost)); diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index d2b855c..1e4f1df 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -150,6 +150,7 @@ std::map pinned_memory; //support for pinned memories added std::map pinned_memory_size; +std::map g_devPtr_Size; int no_of_ptx=0; std::map > version_filename; @@ -487,8 +488,10 @@ __host__ cudaError_t CUDARTAPI cudaMalloc(void **devPtr, size_t size) { CUctx_st* context = GPGPUSim_Context(); *devPtr = context->get_device()->get_gpgpu()->gpu_malloc(size); - if(g_debug_execution >= 3) + if(g_debug_execution >= 3){ printf("GPGPU-Sim PTX: cudaMallocing %zu bytes starting at 0x%llx..\n",size, (unsigned long long) *devPtr); + g_devPtr_Size[*devPtr] = size; + } if ( *devPtr ) { return g_last_cudaError = cudaSuccess; } else { @@ -2374,8 +2377,10 @@ cudaError_t CUDARTAPI cudaHostGetDevicePointer(void **pDevice, void *pHost, unsi assert(i != pinned_memory_size.end()); size_t size = i->second; *pDevice = gpu->gpu_malloc(size); - if(g_debug_execution >= 3) + if(g_debug_execution >= 3){ printf("GPGPU-Sim PTX: cudaMallocing %zu bytes starting at 0x%llx..\n",size, (unsigned long long) *pDevice); + g_devPtr_Size[*pDevice] = size; + } if ( *pDevice ) { pinned_memory[pHost]=pDevice; //Copy contents in cpu to gpu @@ -2617,8 +2622,9 @@ kernel_info_t *gpgpu_cuda_ptx_sim_init_grid( const char *hostFun, g_ptx_kernel_count++; fflush(stdout); + if(g_debug_execution >= 3){ - entry->debug_param(); + entry->debug_param(g_devPtr_Size, result->get_param_memory()); } return result; diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index bdb1e8d..8795e1c 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1226,33 +1226,72 @@ void function_info::list_param( FILE *fout ) const fflush(fout); } -void function_info::debug_param() const +void function_info::debug_param(std::map devPtr_Size, memory_space *param_mem) const { static unsigned long counter = 0; - std::string gpgpusim_path(getenv("GPGPUSIM_ROOT")); - assert(!gpgpusim_path.empty()); - std::string command = "mkdir " + gpgpusim_path + "/debug_tools/WatchYourStep/data"; - system(command.c_str()); - - std::string filename(gpgpusim_path + "/debug_tools/WatchYourStep/data/params.config" + std::to_string(counter++)); std::vector< std::pair > param_data; + std::vector paramIsPointer; + + char * gpgpusim_path = getenv("GPGPUSIM_ROOT"); + assert(gpgpusim_path!=NULL); + std::string command = std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; + system(command.c_str()); + std::string filename(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/params.config" + std::to_string(counter++)); + + //initialize paramList + char buff[1024]; + std::string filename_c(filename+"_c"); + snprintf(buff,1024,"c++filt %s > %s", get_name().c_str(), filename_c.c_str()); + system(buff); + FILE *fp = fopen(filename_c.c_str(), "r"); + fgets(buff, 1024, fp); + fclose(fp); + std::string fn(buff); + size_t pos1, pos2; + pos1 = fn.find("("); + pos2 = fn.find(")"); + assert(pos2>pos1&&pos1>0); + strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); + char *tok; + tok = strtok(buff, ","); + while(tok!=NULL){ + std::string param(tok); + if(param.find("*")!=std::string::npos){ + paramIsPointer.push_back(true); + }else{ + paramIsPointer.push_back(false); + } + tok = strtok(NULL, ","); + } for( std::map::const_iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { const param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); param_t param_value = p.get_value(); - unsigned char val[param_value.size]; - memcpy((void*) val, (void*)((char*)param_value.pdata+param_value.offset), param_value.size); - param_data.push_back(std::pair(param_value.size,val)); +// if (paramIsPointer[i->first]){ +// assert(param_value.size==8); +// }else{ + unsigned char val[param_value.size]; + param_mem->read(param_addr,param_value.size,(void*)val); + param_data.push_back(std::pair(param_value.size,val)); + //} } FILE *fout = fopen (filename.c_str(), "w"); fprintf(fout, "%s\n", get_name().c_str()); + size_t index = 0; for( std::vector< std::pair >::const_iterator i=param_data.begin(); i!=param_data.end(); i++ ) { + if (paramIsPointer[index]){ + fprintf(fout, "*"); + } fprintf(fout, "%lu :", i->first); for (size_t j = 0; jfirst; j++){ fprintf(fout, " %u", i->second[j]); } fprintf(fout, "\n"); + index++; } fflush(fout); fclose(fout); diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 341f9b7..43907d3 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1257,7 +1257,7 @@ public: void finalize( memory_space *param_mem ); void param_to_shared( memory_space *shared_mem, symbol_table *symtab ); void list_param( FILE *fout ) const; - void debug_param() const; + void debug_param(std::map devPtr_Size, memory_space *param_mem) const; const struct gpgpu_ptx_sim_info* get_kernel_info () const { -- cgit v1.3 From 87cc31c53123f918af3250085da72a3ab8361cf5 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Thu, 28 Jun 2018 13:13:59 -0700 Subject: Tests to find conditions that a value is a pointer and new mallocPtr_Size --- libcuda/cuda_runtime_api.cc | 15 +++++++++------ src/cuda-sim/cuda-sim.cc | 23 +++++++++++++++++++---- src/cuda-sim/ptx_ir.h | 2 +- 3 files changed, 29 insertions(+), 11 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index 1e4f1df..28ff1c4 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -150,7 +150,7 @@ std::map pinned_memory; //support for pinned memories added std::map pinned_memory_size; -std::map g_devPtr_Size; +std::map g_mallocPtr_Size; int no_of_ptx=0; std::map > version_filename; @@ -322,8 +322,8 @@ public: } kernel_config() { - m_GridDim=NULL; - m_BlockDim=NULL; + m_GridDim=dim3(-1,-1,-1); + m_BlockDim=dim3(-1,-1,-1); m_sharedMem=0; m_stream =NULL; } @@ -490,7 +490,7 @@ __host__ cudaError_t CUDARTAPI cudaMalloc(void **devPtr, size_t size) *devPtr = context->get_device()->get_gpgpu()->gpu_malloc(size); if(g_debug_execution >= 3){ printf("GPGPU-Sim PTX: cudaMallocing %zu bytes starting at 0x%llx..\n",size, (unsigned long long) *devPtr); - g_devPtr_Size[*devPtr] = size; + g_mallocPtr_Size[(unsigned long long)*devPtr] = size; } if ( *devPtr ) { return g_last_cudaError = cudaSuccess; @@ -1075,6 +1075,9 @@ __host__ cudaError_t CUDARTAPI cudaSetupArgument(const void *arg, size_t size, s gpgpusim_ptx_assert( !g_cuda_launch_stack.empty(), "empty launch stack" ); kernel_config &config = g_cuda_launch_stack.back(); config.set_arg(arg,size,offset); + printf("GPGPU-Sim PTX: Setting up arguments for %zu bytes starting at 0x%llx..\n",size, (unsigned long long) arg); + assert(size!=8||g_mallocPtr_Size.find(*(long long*)arg)!=g_mallocPtr_Size.end()); + //assert((*(long long*)arg)==0||g_mallocPtr_Size.find(*(long long*)arg)!=g_mallocPtr_Size.end()); return g_last_cudaError = cudaSuccess; } @@ -2379,7 +2382,7 @@ cudaError_t CUDARTAPI cudaHostGetDevicePointer(void **pDevice, void *pHost, unsi *pDevice = gpu->gpu_malloc(size); if(g_debug_execution >= 3){ printf("GPGPU-Sim PTX: cudaMallocing %zu bytes starting at 0x%llx..\n",size, (unsigned long long) *pDevice); - g_devPtr_Size[*pDevice] = size; + g_mallocPtr_Size[(unsigned long long)*pDevice] = size; } if ( *pDevice ) { pinned_memory[pHost]=pDevice; @@ -2624,7 +2627,7 @@ kernel_info_t *gpgpu_cuda_ptx_sim_init_grid( const char *hostFun, if(g_debug_execution >= 3){ - entry->debug_param(g_devPtr_Size, result->get_param_memory()); + entry->debug_param(g_mallocPtr_Size, result->get_param_memory()); } return result; diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 8795e1c..ab607e0 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1226,7 +1226,7 @@ void function_info::list_param( FILE *fout ) const fflush(fout); } -void function_info::debug_param(std::map devPtr_Size, memory_space *param_mem) const +void function_info::debug_param(std::map mallocPtr_Size, memory_space *param_mem) const { static unsigned long counter = 0; std::vector< std::pair > param_data; @@ -1248,8 +1248,8 @@ void function_info::debug_param(std::map devPtr_Size, memory_spa fclose(fp); std::string fn(buff); size_t pos1, pos2; - pos1 = fn.find("("); - pos2 = fn.find(")"); + pos1 = fn.find_last_of("("); + pos2 = fn.find(")", pos1); assert(pos2>pos1&&pos1>0); strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); char *tok; @@ -1270,8 +1270,23 @@ void function_info::debug_param(std::map devPtr_Size, memory_spa symbol *param = m_symtab->lookup(name.c_str()); addr_t param_addr = param->get_address(); param_t param_value = p.get_value(); + + if (paramIsPointer[i->first]){ + assert(param_value.size==8&&mallocPtr_Size.find(*(long long*)param_value.pdata)!=mallocPtr_Size.end()); + }else{ + assert(!(param_value.size==8&&mallocPtr_Size.find(*(long long*)param_value.pdata)!=mallocPtr_Size.end())); + } // if (paramIsPointer[i->first]){ // assert(param_value.size==8); +// void *array_pointer = (void*)*val; +// assert(devPtr_Size.find(array_pointer)!=devPtr_Size.end()); +// size_t array_size = devPtr_Size[array_pointer]; +// unsigned char array_val[array_size]; +// memcpy((void*) array_val, array_pointer, array_size); +// param_data.push_back(std::pair(array_size,array_val)); +// +// assert(size!=8||g_mallocPtr_Size.find(*(long long*)arg)!=g_mallocPtr_Size.end()); +// // }else{ unsigned char val[param_value.size]; param_mem->read(param_addr,param_value.size,(void*)val); @@ -1295,7 +1310,7 @@ void function_info::debug_param(std::map devPtr_Size, memory_spa } fflush(fout); fclose(fout); - exit(0); + //exit(0); } template diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 43907d3..1eb9d65 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1257,7 +1257,7 @@ public: void finalize( memory_space *param_mem ); void param_to_shared( memory_space *shared_mem, symbol_table *symtab ); void list_param( FILE *fout ) const; - void debug_param(std::map devPtr_Size, memory_space *param_mem) const; + void debug_param(std::map mallocPtr_Size, memory_space *param_mem) const; const struct gpgpu_ptx_sim_info* get_kernel_info () const { -- cgit v1.3 From df58abd050457f062a2ecfe6202c85be5b939230 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Thu, 28 Jun 2018 14:59:07 -0700 Subject: dumps pointers by accessing global memory --- libcuda/cuda_runtime_api.cc | 4 +-- src/cuda-sim/cuda-sim.cc | 59 +++++++++------------------------------------ src/cuda-sim/ptx_ir.h | 2 +- 3 files changed, 14 insertions(+), 51 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index 28ff1c4..53b4da8 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -1077,7 +1077,6 @@ __host__ cudaError_t CUDARTAPI cudaSetupArgument(const void *arg, size_t size, s config.set_arg(arg,size,offset); printf("GPGPU-Sim PTX: Setting up arguments for %zu bytes starting at 0x%llx..\n",size, (unsigned long long) arg); assert(size!=8||g_mallocPtr_Size.find(*(long long*)arg)!=g_mallocPtr_Size.end()); - //assert((*(long long*)arg)==0||g_mallocPtr_Size.find(*(long long*)arg)!=g_mallocPtr_Size.end()); return g_last_cudaError = cudaSuccess; } @@ -2624,10 +2623,9 @@ kernel_info_t *gpgpu_cuda_ptx_sim_init_grid( const char *hostFun, entry->finalize(result->get_param_memory()); g_ptx_kernel_count++; fflush(stdout); - if(g_debug_execution >= 3){ - entry->debug_param(g_mallocPtr_Size, result->get_param_memory()); + entry->debug_param(g_mallocPtr_Size, result->get_param_memory(), (gpgpu_t *) context->get_device()->get_gpgpu()); } return result; diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index ab607e0..bade750 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1226,7 +1226,7 @@ void function_info::list_param( FILE *fout ) const fflush(fout); } -void function_info::debug_param(std::map mallocPtr_Size, memory_space *param_mem) const +void function_info::debug_param(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu) const { static unsigned long counter = 0; std::vector< std::pair > param_data; @@ -1234,36 +1234,10 @@ void function_info::debug_param(std::map mallocPtr_S char * gpgpusim_path = getenv("GPGPUSIM_ROOT"); assert(gpgpusim_path!=NULL); - std::string command = std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; - system(command.c_str()); +// std::string command = std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; +// system(command.c_str()); std::string filename(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/params.config" + std::to_string(counter++)); - //initialize paramList - char buff[1024]; - std::string filename_c(filename+"_c"); - snprintf(buff,1024,"c++filt %s > %s", get_name().c_str(), filename_c.c_str()); - system(buff); - FILE *fp = fopen(filename_c.c_str(), "r"); - fgets(buff, 1024, fp); - fclose(fp); - std::string fn(buff); - size_t pos1, pos2; - pos1 = fn.find_last_of("("); - pos2 = fn.find(")", pos1); - assert(pos2>pos1&&pos1>0); - strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); - char *tok; - tok = strtok(buff, ","); - while(tok!=NULL){ - std::string param(tok); - if(param.find("*")!=std::string::npos){ - paramIsPointer.push_back(true); - }else{ - paramIsPointer.push_back(false); - } - tok = strtok(NULL, ","); - } - for( std::map::const_iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { const param_info &p = i->second; std::string name = p.get_name(); @@ -1271,30 +1245,22 @@ void function_info::debug_param(std::map mallocPtr_S addr_t param_addr = param->get_address(); param_t param_value = p.get_value(); - if (paramIsPointer[i->first]){ - assert(param_value.size==8&&mallocPtr_Size.find(*(long long*)param_value.pdata)!=mallocPtr_Size.end()); + if(param_value.size==8&&mallocPtr_Size.find(*(unsigned long long*)param_value.pdata)!=mallocPtr_Size.end()){ + size_t array_size = mallocPtr_Size[*(unsigned long long*)param_value.pdata]; + unsigned char val[array_size]; + gpu->get_global_memory()->read(param_addr,array_size,(void*)val); + param_data.push_back(std::pair(array_size,val)); + paramIsPointer.push_back(true); }else{ - assert(!(param_value.size==8&&mallocPtr_Size.find(*(long long*)param_value.pdata)!=mallocPtr_Size.end())); - } -// if (paramIsPointer[i->first]){ -// assert(param_value.size==8); -// void *array_pointer = (void*)*val; -// assert(devPtr_Size.find(array_pointer)!=devPtr_Size.end()); -// size_t array_size = devPtr_Size[array_pointer]; -// unsigned char array_val[array_size]; -// memcpy((void*) array_val, array_pointer, array_size); -// param_data.push_back(std::pair(array_size,array_val)); -// -// assert(size!=8||g_mallocPtr_Size.find(*(long long*)arg)!=g_mallocPtr_Size.end()); -// -// }else{ unsigned char val[param_value.size]; param_mem->read(param_addr,param_value.size,(void*)val); param_data.push_back(std::pair(param_value.size,val)); - //} + paramIsPointer.push_back(false); + } } FILE *fout = fopen (filename.c_str(), "w"); + printf("Writing data to %s ...\n", filename.c_str()); fprintf(fout, "%s\n", get_name().c_str()); size_t index = 0; for( std::vector< std::pair >::const_iterator i=param_data.begin(); i!=param_data.end(); i++ ) { @@ -1310,7 +1276,6 @@ void function_info::debug_param(std::map mallocPtr_S } fflush(fout); fclose(fout); - //exit(0); } template diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 1eb9d65..449d615 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1257,7 +1257,7 @@ public: void finalize( memory_space *param_mem ); void param_to_shared( memory_space *shared_mem, symbol_table *symtab ); void list_param( FILE *fout ) const; - void debug_param(std::map mallocPtr_Size, memory_space *param_mem) const; + void debug_param(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu) const; const struct gpgpu_ptx_sim_info* get_kernel_info () const { -- cgit v1.3 From 1fddb06b153a3e5fbc255e89bb6861cce1319039 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Wed, 4 Jul 2018 12:52:42 -0700 Subject: dump and load block and grid size and launch --- .../WatchYourStep/ptxjitplus/ptxjitplus.cpp | 14 ++++++------- libcuda/cuda_runtime_api.cc | 24 ++++++++++++++++++++-- src/cuda-sim/cuda-sim.cc | 3 ++- src/cuda-sim/ptx_ir.h | 2 +- 4 files changed, 32 insertions(+), 11 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp b/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp index 65dfa84..df0ff8d 100644 --- a/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp +++ b/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp @@ -40,6 +40,7 @@ const char *sSDKname = "PTX Just In Time (JIT) Compilation (no-qatest)"; char *wys_exec_path; char *wys_exec_name; char *wys_launch_num; +dim3 gridDim, blockDim; std::string kernelName; void ptxJIT(int argc, char **argv, CUmodule *phModule, CUfunction *phKernel, CUlinkState *lState) @@ -133,6 +134,7 @@ void initializeData(std::vector& param_data, std::vector< std::p printf("Processing :%s ...\n", buff); fflush(stdout); kernelName = std::string(buff); + fscanf(fin, "%u,%u,%u %u,%u,%u\n", &gridDim.x, &gridDim.y, &gridDim.z, &blockDim.x, &blockDim.y, &blockDim.z); //fill data structure to pass in params later while (!feof(fin)){ std::pair info; @@ -248,12 +250,6 @@ int main(int argc, char **argv) ptxJIT(argc, argv, &hModule, &hKernel, &lState); checkCudaErrors(cudaFree(d_tmp)); - - // Set the kernel parameters (Driver API) - - // TODO: automatically load these values in - checkCudaErrors(cuFuncSetBlockShape(hKernel, 128, 1, 1)); - //maps param number to pointer to device data std::map< size_t, unsigned char* > m_device_data; //Initialize param_data for kernel @@ -275,7 +271,8 @@ int main(int argc, char **argv) // Launch the kernel (Driver API_) // TODO: automatically load these values in - checkCudaErrors(cuLaunchGrid(hKernel, 1, 20)); + CUDAAPI cuLaunchKernel(hKernel, gridDim.x, gridDim.y, gridDim.z, blockDim.x, blockDim.y, blockDim.z, + 0, NULL, NULL, NULL); std::cout << "CUDA kernel launched" << std::endl; //maps param number to pointer to output data @@ -299,6 +296,9 @@ int main(int argc, char **argv) fprintf(fout, "param %zu: size = %zu, data = ", i->first,param_info[i->first].first); for (size_t j = 0; jfirst].first; j++){ fprintf(fout, " %u", i->second[j]); + if (j&&(!(j%20))){ + fprintf(fout, "\n"); + } } fprintf(fout, "\n"); } diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index cad4d87..410f15f 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -2624,7 +2624,7 @@ kernel_info_t *gpgpu_cuda_ptx_sim_init_grid( const char *hostFun, fflush(stdout); if(g_debug_execution >= 3){ - entry->debug_param(g_mallocPtr_Size, result->get_param_memory(), (gpgpu_t *) context->get_device()->get_gpgpu()); + entry->debug_param(g_mallocPtr_Size, result->get_param_memory(), (gpgpu_t *) context->get_device()->get_gpgpu(), gridDim, blockDim); } return result; @@ -2744,7 +2744,6 @@ CUresult CUDAAPI cuParamSetv(CUfunction hfunc, int offset, void *ptr, unsigned i CUresult CUDAAPI cuLaunchGrid(CUfunction f, int grid_width, int grid_height) { - CUctx_st* context = GPGPUSim_Context(); const char *hostFun = (const char*) f; gpgpusim_ptx_assert( !g_cuda_launch_stack.empty(), "empty launch stack" ); kernel_config &config = g_cuda_launch_stack.back(); @@ -2754,4 +2753,25 @@ CUresult CUDAAPI cuLaunchGrid(CUfunction f, int grid_width, int grid_height) cudaLaunch(hostFun); return CUDA_SUCCESS; } + + +CUresult CUDAAPI cuLaunchKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, + unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, + unsigned int sharedMemBytes, CUstream hStream, void **kernelParams, void **extra) +{ + if (sharedMemBytes!=0||hStream!=NULL||kernelParams!=NULL||extra!=NULL){ + printf("GPGPU-Sim CUDA DRIVER API: Warning: Currently do not support \nsharedMemBytes, hStream, kernelParams, and extra parameters.\n"); + } + const char *hostFun = (const char*) f; + gpgpusim_ptx_assert( !g_cuda_launch_stack.empty(), "empty launch stack" ); + kernel_config &config = g_cuda_launch_stack.back(); + dim3 *d_b = new dim3(blockDimX, blockDimY, blockDimZ); + config.set_block_dim(d_b); + dim3 *d_g = new dim3(gridDimX, gridDimY, gridDimZ); + config.set_grid_dim(d_g); + + cudaLaunch(hostFun); + return CUDA_SUCCESS; +} + #endif diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 62aef6d..3003c4c 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1226,7 +1226,7 @@ void function_info::list_param( FILE *fout ) const fflush(fout); } -void function_info::debug_param(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu) const +void function_info::debug_param(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) const { static unsigned long counter = 0; std::vector< std::pair > param_data; @@ -1265,6 +1265,7 @@ void function_info::debug_param(std::map mallocPtr_S FILE *fout = fopen (filename.c_str(), "w"); printf("Writing data to %s ...\n", filename.c_str()); fprintf(fout, "%s\n", get_name().c_str()); + fprintf(fout, "%u,%u,%u %u,%u,%u\n", gridDim.x, gridDim.y, gridDim.z, blockDim.x, blockDim.y, blockDim.z); size_t index = 0; for( std::vector< std::pair >::const_iterator i=param_data.begin(); i!=param_data.end(); i++ ) { if (paramIsPointer[index]){ diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 449d615..b29621c 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1257,7 +1257,7 @@ public: void finalize( memory_space *param_mem ); void param_to_shared( memory_space *shared_mem, symbol_table *symtab ); void list_param( FILE *fout ) const; - void debug_param(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu) const; + void debug_param(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) const; const struct gpgpu_ptx_sim_info* get_kernel_info () const { -- cgit v1.3 From 7c4f9d7bff7b1725f3da6ddc8abf3f77a1cbe6f5 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Wed, 4 Jul 2018 16:41:20 -0700 Subject: dumps ptx kernels and scripts to launch all kernels --- debug_tools/WatchYourStep/ptxjitplus/launchkernels | 3 ++ .../WatchYourStep/ptxjitplus/ptxjitplus.cpp | 10 +++--- libcuda/cuda_runtime_api.cc | 2 +- src/cuda-sim/cuda-sim.cc | 37 ++++++++++++++++++++-- src/cuda-sim/ptx_ir.h | 2 +- 5 files changed, 44 insertions(+), 10 deletions(-) create mode 100644 debug_tools/WatchYourStep/ptxjitplus/launchkernels (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/debug_tools/WatchYourStep/ptxjitplus/launchkernels b/debug_tools/WatchYourStep/ptxjitplus/launchkernels new file mode 100644 index 0000000..d2fd015 --- /dev/null +++ b/debug_tools/WatchYourStep/ptxjitplus/launchkernels @@ -0,0 +1,3 @@ +#Launches kernels from $1 to $2 +#Note must source this script +for num in $(eval echo {$1..$2}); do export WYS_LAUNCH_NUM=$num; echo Launching kernel $num...; ./ptxjitplus; done diff --git a/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp b/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp index df0ff8d..1f094ba 100644 --- a/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp +++ b/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp @@ -81,18 +81,16 @@ void ptxJIT(int argc, char **argv, CUmodule *phModule, CUfunction *phKernel, CUl if (sizeof(void *)==4) { // Load the PTX from the string myPtx32 - printf("Loading myPtx32[] program\n"); - // PTX May also be loaded from file, as per below. - myErr = cuLinkAddData(*lState, CU_JIT_INPUT_PTX, (void *)myPtx32, strlen(myPtx32)+1, 0, 0, 0, 0); + printf("Loading myPtx32[] program...\n"); + printf("WARNING: 32-bit execution is untested"); } else { // Load the PTX from the string myPtx (64-bit) printf("Loading myPtx[] program\n"); - //myErr = cuLinkAddData(*lState, CU_JIT_INPUT_PTX, (void *)myPtx64, strlen(myPtx64)+1, 0, 0, 0, 0); - // PTX May also be loaded from file, as per below. - myErr = cuLinkAddFile(*lState, CU_JIT_INPUT_PTX, "myPtx64.ptx",0,0,0); } + std::string ptx_file (std::string("../data/ptx.config") + wys_launch_num); + myErr = cuLinkAddFile(*lState, CU_JIT_INPUT_PTX, ptx_file.c_str(),0,0,0); if (myErr != CUDA_SUCCESS) { diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index 410f15f..b67ea85 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -2624,7 +2624,7 @@ kernel_info_t *gpgpu_cuda_ptx_sim_init_grid( const char *hostFun, fflush(stdout); if(g_debug_execution >= 3){ - entry->debug_param(g_mallocPtr_Size, result->get_param_memory(), (gpgpu_t *) context->get_device()->get_gpgpu(), gridDim, blockDim); + entry->ptx_jit_config(g_mallocPtr_Size, result->get_param_memory(), (gpgpu_t *) context->get_device()->get_gpgpu(), gridDim, blockDim); } return result; diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 3003c4c..fc4f82a 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1226,7 +1226,7 @@ void function_info::list_param( FILE *fout ) const fflush(fout); } -void function_info::debug_param(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) const +void function_info::ptx_jit_config(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) const { static unsigned long counter = 0; std::vector< std::pair > param_data; @@ -1236,7 +1236,7 @@ void function_info::debug_param(std::map mallocPtr_S assert(gpgpusim_path!=NULL); std::string command = std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; system(command.c_str()); - std::string filename(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/params.config" + std::to_string(counter++)); + std::string filename(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/params.config" + std::to_string(counter)); for( std::map::const_iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { const param_info &p = i->second; @@ -1280,6 +1280,39 @@ void function_info::debug_param(std::map mallocPtr_S } fflush(fout); fclose(fout); + + //ptx config + char buff[1024]; + std::string ptx_config_fn(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/ptx.config" + std::to_string(counter)); + snprintf(buff, 1024, "grep -rn \".entry %s\" *.ptx | cut -d \":\" -f 1-2 > %s", get_name().c_str(), ptx_config_fn.c_str()); + system(buff); + FILE *fin = fopen(ptx_config_fn.c_str(), "r"); + char ptx_source[256]; + unsigned line_number; + int numscanned = fscanf(fin, "%[^:]:%u", ptx_source, &line_number); + assert(numscanned == 2); + fclose(fin); + snprintf(buff, 1024, "grep -rn \".version\" %s | cut -d \":\" -f 1 | xargs -I \"{}\" awk \"NR>={}&&NR<={}+2\" %s > %s", ptx_source, ptx_source, ptx_config_fn.c_str()); + system(buff); + fin = fopen(ptx_source, "r"); + assert(fin!=NULL); + fout = fopen(ptx_config_fn.c_str(), "a"); + assert(fout!=NULL); + for (unsigned i = 0; i diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index b29621c..e0b5e96 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1257,7 +1257,7 @@ public: void finalize( memory_space *param_mem ); void param_to_shared( memory_space *shared_mem, symbol_table *symtab ); void list_param( FILE *fout ) const; - void debug_param(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) const; + void ptx_jit_config(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) const; const struct gpgpu_ptx_sim_info* get_kernel_info () const { -- cgit v1.3 From 3a09960577b9c9cbcce8fedeef8874ccc533f378 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Wed, 18 Jul 2018 17:14:41 -0700 Subject: added c++filt that works better, param offset dumping, protection for failing system calls --- .../WatchYourStep/ptxjitplus/ptxjitplus.cpp | 2 +- libcuda/cuda_runtime_api.cc | 5 +- src/cuda-sim/cuda-sim.cc | 78 +++++++++++++++++++--- src/cuda-sim/ptx_ir.h | 2 +- 4 files changed, 74 insertions(+), 13 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp b/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp index b7f9f2d..26a6d29 100644 --- a/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp +++ b/debug_tools/WatchYourStep/ptxjitplus/ptxjitplus.cpp @@ -183,6 +183,7 @@ void initializeData(std::vector& param_data, std::vector< std::p assert( err==1 ); params[i] = (unsigned char) val; } + //TODO: parse param offset param_info.push_back(info); param_data.push_back(params); err = fscanf(fin, "\n"); @@ -290,7 +291,6 @@ int main(int argc, char **argv) checkCudaErrors(cuParamSetSize(hKernel, paramOffset)); // Launch the kernel (Driver API_) - // TODO: automatically load these values in CUDAAPI cuLaunchKernel(hKernel, gridDim.x, gridDim.y, gridDim.z, blockDim.x, blockDim.y, blockDim.z, 0, NULL, paramKernels, NULL); std::cout << "CUDA kernel launched" << std::endl; diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index 866fa3b..db4f58e 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -1797,7 +1797,10 @@ void extract_code_using_cuobjdump(){ //Running cuobjdump using dynamic link to current process snprintf(command,1000,"md5sum %s ", app_binary.c_str()); printf("Running md5sum using \"%s\"\n", command); - system(command); + if(system(command)){ + std::cout << "Failed to execute: " << command << std::endl; + exit(1); + } // Running cuobjdump using dynamic link to current process // Needs the option '-all' to extract PTX from CDP-enabled binary extern bool g_cdp_enabled; diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index d2f096f..a499ce9 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1226,10 +1226,11 @@ void function_info::list_param( FILE *fout ) const fflush(fout); } -void function_info::ptx_jit_config(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) const +void function_info::ptx_jit_config(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) { static unsigned long counter = 0; std::vector< std::pair > param_data; + std::vector offsets; std::vector paramIsPointer; char * gpgpusim_path = getenv("GPGPUSIM_ROOT"); @@ -1237,18 +1238,67 @@ void function_info::ptx_jit_config(std::map mallocPt char * wys_exec_path = getenv("WYS_EXEC_PATH"); assert(wys_exec_path!=NULL); std::string command = std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; - system(command.c_str()); + if(system(command.c_str())!=0){ + printf("WARNING: Failed to execute mkdir \n"); + printf("Problematic call: %s", command.c_str()); + abort(); + } std::string filename(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/params.config" + std::to_string(counter)); - for( std::map::const_iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - const param_info &p = i->second; + //initialize paramList + char buff[1024]; + std::string filename_c(filename+"_c"); + snprintf(buff,1024,"c++filt %s > %s", get_name().c_str(), filename_c.c_str()); + system(buff); + FILE *fp = fopen(filename_c.c_str(), "r"); + fgets(buff, 1024, fp); + fclose(fp); + std::string fn(buff); + size_t pos1, pos2; + pos1 = fn.find_last_of("("); + pos2 = fn.find(")", pos1); + assert(pos2>pos1&&pos1>0); + strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); + char *tok; + tok = strtok(buff, ","); + std::string tmp; + while(tok!=NULL){ + std::string param(tok); + if(param.find("<")!=std::string::npos){ + assert(param.find(">")==std::string::npos); + assert(param.find("*")==std::string::npos); + tmp = param; + } else { + if (tmp.length()>0){ + tmp = ""; + assert(param.find(">")!=std::string::npos); + assert(param.find("<")==std::string::npos); + assert(param.find("*")==std::string::npos); + } + if(param.find("*")!=std::string::npos){ + paramIsPointer.push_back(true); + }else{ + paramIsPointer.push_back(false); + } + } + tok = strtok(NULL, ","); + } + + + for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + param_info &p = i->second; std::string name = p.get_name(); symbol *param = m_symtab->lookup(name.c_str()); addr_t param_addr = param->get_address(); param_t param_value = p.get_value(); + offsets.push_back((unsigned)p.get_offset()); - if(param_value.size==sizeof(void*) && mallocPtr_Size.find(*(unsigned long long*)param_value.pdata)!=mallocPtr_Size.end()){ + if (paramIsPointer[i->first]){ //is pointer + assert(param_value.size==8); + assert(param_value.size==sizeof(void*) && mallocPtr_Size.find(*(unsigned long long*)param_value.pdata)!=mallocPtr_Size.end()); + //TODO: check in middle of malloc'd memory for pointer + size_t array_size = mallocPtr_Size[*(unsigned long long*)param_value.pdata]; unsigned char* val = (unsigned char*) malloc(param_value.size); param_mem->read(param_addr,param_value.size,(void*)val); @@ -1257,7 +1307,7 @@ void function_info::ptx_jit_config(std::map mallocPt param_data.push_back(std::pair(array_size,array_val)); paramIsPointer.push_back(true); }else{ - unsigned char val[param_value.size]; + unsigned char* val = (unsigned char*) malloc(param_value.size); param_mem->read(param_addr,param_value.size,(void*)val); param_data.push_back(std::pair(param_value.size,val)); paramIsPointer.push_back(false); @@ -1277,6 +1327,8 @@ void function_info::ptx_jit_config(std::map mallocPt for (size_t j = 0; jfirst; j++){ fprintf(fout, " %u", i->second[j]); } + fprintf(fout, " : %u", offsets[index]); + free (i->second); fprintf(fout, "\n"); index++; } @@ -1284,10 +1336,13 @@ void function_info::ptx_jit_config(std::map mallocPt fclose(fout); //ptx config - char buff[1024]; std::string ptx_config_fn(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/ptx.config" + std::to_string(counter)); snprintf(buff, 1024, "grep -rn \".entry %s\" %s/*.ptx | cut -d \":\" -f 1-2 > %s", get_name().c_str(), wys_exec_path, ptx_config_fn.c_str()); - system(buff); + if (system(buff)!=0){ + printf("WARNING: Failed to execute grep to find ptx source \n"); + printf("Problematic call: %s", buff); + abort(); + } FILE *fin = fopen(ptx_config_fn.c_str(), "r"); char ptx_source[256]; unsigned line_number; @@ -1295,7 +1350,11 @@ void function_info::ptx_jit_config(std::map mallocPt assert(numscanned == 2); fclose(fin); snprintf(buff, 1024, "grep -rn \".version\" %s | cut -d \":\" -f 1 | xargs -I \"{}\" awk \"NR>={}&&NR<={}+2\" %s > %s", ptx_source, ptx_source, ptx_config_fn.c_str()); - system(buff); + if (system(buff)!=0){ + printf("WARNING: Failed to execute grep to find ptx header \n"); + printf("Problematic call: %s", buff); + abort(); + } fin = fopen(ptx_source, "r"); assert(fin!=NULL); printf("Writing data to %s ...\n", ptx_config_fn.c_str()); @@ -1318,7 +1377,6 @@ void function_info::ptx_jit_config(std::map mallocPt fflush(fout); fclose(fout); counter++; - //TODO: Free param_data } template diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index e0b5e96..ef4cf48 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1257,7 +1257,7 @@ public: void finalize( memory_space *param_mem ); void param_to_shared( memory_space *shared_mem, symbol_table *symtab ); void list_param( FILE *fout ) const; - void ptx_jit_config(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) const; + void ptx_jit_config(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) ; const struct gpgpu_ptx_sim_info* get_kernel_info () const { -- cgit v1.3 From 06147dc4a1ed08f41be159bba09a7a6c9bd7318f Mon Sep 17 00:00:00 2001 From: aamir Date: Sun, 22 Jul 2018 18:33:32 -0700 Subject: added regression and debugged the failing testcase --- cutlass-example/Makefile | 11 ++++- cutlass-example/cutlass_example.cu | 96 +++++++++++++++++++++++++++++++++++++- cutlass-example/launch.sh | 39 ++++++++++++++++ src/cuda-sim/ptx_ir.h | 12 ++++- 4 files changed, 153 insertions(+), 5 deletions(-) create mode 100755 cutlass-example/launch.sh (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/cutlass-example/Makefile b/cutlass-example/Makefile index 8156847..f72b732 100644 --- a/cutlass-example/Makefile +++ b/cutlass-example/Makefile @@ -34,4 +34,13 @@ INC_DIR = ./ LDFLAGS = --gpu-architecture=compute_70 --gpu-code=compute_70 -lcudart -I$(INC_DIR) myprog: $(obj) - $(CC) -o $@ $^ $(LDFLAGS) + $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) + +clean: + rm -rf gemm_tt_* + rm -rf host_results_* + rm -rf gpgpusim_power_report__* + rm -rf _cuobjdump* + rm -rf gpgpu_inst_stats.txt + rm -rf myprog + echo Clean done diff --git a/cutlass-example/cutlass_example.cu b/cutlass-example/cutlass_example.cu index d3e8f89..6371164 100644 --- a/cutlass-example/cutlass_example.cu +++ b/cutlass-example/cutlass_example.cu @@ -6,11 +6,103 @@ #include int main(int argc, char* argv[]) { - typedef cutlass::gemm::WmmaGemmTraits > + WmmaGemmTraits; + run_gemm(16, 16, 16); +#endif + +#ifdef WMMA_GEMM_16x16x32_NT + typedef cutlass::gemm::WmmaGemmTraits > + WmmaGemmTraits; + run_gemm(16, 16, 32); +#endif + +#ifdef WMMA_GEMM_16x16x16_NN + typedef cutlass::gemm::WmmaGemmTraits > + WmmaGemmTraits; + run_gemm(16, 16, 16); +#endif + + +#ifdef WMMA_GEMM_16x16x32_NN + typedef cutlass::gemm::WmmaGemmTraits > + WmmaGemmTraits; + run_gemm(16, 16, 32); +#endif + +#ifdef WMMA_GEMM_16x16x16_TT + typedef cutlass::gemm::WmmaGemmTraits > + WmmaGemmTraits; + run_gemm(16, 16, 16); +#endif + +#ifdef WMMA_GEMM_16x16x32_TT + typedef cutlass::gemm::WmmaGemmTraits > + WmmaGemmTraits; + run_gemm(16, 16, 32); +#endif + +#ifdef WMMA_GEMM_16x16x16_TN + typedef cutlass::gemm::WmmaGemmTraits > + WmmaGemmTraits; + run_gemm(16, 16, 16); +#endif + +#ifdef WMMA_GEMM_16x16x32_TN + typedef cutlass::gemm::WmmaGemmTraits > + WmmaGemmTraits; + run_gemm(16, 16, 32); +#endif + +#ifdef WMMA_16x16x16_GEMM_256x256x128_NT + typedef cutlass::gemm::WmmaGemmTraits > + WmmaGemmTraits; + run_gemm(256, 256, 128); +#endif + +#ifdef WMMA_16x16x16_GEMM_256x256x128_NN + typedef cutlass::gemm::WmmaGemmTraits > + WmmaGemmTraits; + run_gemm(256, 256, 128); +#endif + +#ifdef WMMA_16x16x16_GEMM_256x256x128_TT + typedef cutlass::gemm::WmmaGemmTraits > + WmmaGemmTraits; + run_gemm(256, 256, 128); +#endif + +#ifdef WMMA_16x16x16_GEMM_256x256x128_TN + typedef cutlass::gemm::WmmaGemmTraits > WmmaGemmTraits; run_gemm(256, 256, 128); +#endif } diff --git a/cutlass-example/launch.sh b/cutlass-example/launch.sh new file mode 100755 index 0000000..2cae877 --- /dev/null +++ b/cutlass-example/launch.sh @@ -0,0 +1,39 @@ +#!/bin/bash +make clean +make CFLAGS=-DWMMA_GEMM_16x16x16_NT +./myprog>LOG_WMMA_GEMM_16x16x16_NT +make clean +make CFLAGS=-DWMMA_GEMM_16x16x32_NT +./myprog>LOG_WMMA_GEMM_16x16x32_NT +make clean +make CFLAGS=-DWMMA_GEMM_16x16x16_NN +./myprog>LOG_WMMA_GEMM_16x16x16_NN +make clean +make CFLAGS=-DWMMA_GEMM_16x16x32_NN +./myprog>LOG_WMMA_GEMM_16x16x32_NN +make clean +make CFLAGS=-DWMMA_GEMM_16x16x16_TT +./myprog>LOG_WMMA_GEMM_16x16x16_TT +make clean +make CFLAGS=-DWMMA_GEMM_16x16x32_TT +./myprog>LOG_WMMA_GEMM_16x16x32_TT +make clean +make CFLAGS=-DWMMA_GEMM_16x16x16_TN +./myprog>LOG_WMMA_GEMM_16x16x16_TN +make clean +make CFLAGS=-DWMMA_GEMM_16x16x32_TN +./myprog>LOG_WMMA_GEMM_16x16x32_TN +make clean +make CFLAGS=-DWMMA_16x16x16_GEMM_256x256x128_NT +./myprog>LOG_WMMA_16x16x16_GEMM_256x256x128_NT +make clean +make CFLAGS=-DWMMA_16x16x16_GEMM_256x256x128_NN +./myprog>LOG_WMMA_16x16x16_GEMM_256x256x128_NN +make clean +make CFLAGS=-DWMMA_16x16x16_GEMM_256x256x128_TT +./myprog>LOG_WMMA_16x16x16_GEMM_256x256x128_TT +make clean +make CFLAGS=-DWMMA_16x16x16_GEMM_256x256x128_TN +./myprog>LOG_WMMA_16x16x16_GEMM_256x256x128_TN +make clean +grep "Result Verifier" LOG_* | wc -l diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 62d7c7c..cff312b 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -434,11 +434,15 @@ public: m_uid = get_uid(); m_valid = true; m_type = memory_t; - m_value.m_vector_symbolic = new const symbol*[4]; + m_value.m_vector_symbolic = new const symbol*[8]; m_value.m_vector_symbolic[0] = addr1; m_value.m_vector_symbolic[1] = addr2; m_value.m_vector_symbolic[2] = NULL; m_value.m_vector_symbolic[3] = NULL; + m_value.m_vector_symbolic[4] = NULL; + m_value.m_vector_symbolic[5] = NULL; + m_value.m_vector_symbolic[6] = NULL; + m_value.m_vector_symbolic[7] = NULL; m_addr_offset = 0; m_vector = false; m_neg_pred = false; @@ -572,11 +576,15 @@ public: m_valid = true; m_vector = true; m_type = vector_t; - m_value.m_vector_symbolic = new const symbol*[4]; + m_value.m_vector_symbolic = new const symbol*[8]; m_value.m_vector_symbolic[0] = s1; m_value.m_vector_symbolic[1] = s2; m_value.m_vector_symbolic[2] = s3; m_value.m_vector_symbolic[3] = s4; + m_value.m_vector_symbolic[4] = NULL; + m_value.m_vector_symbolic[5] = NULL; + m_value.m_vector_symbolic[6] = NULL; + m_value.m_vector_symbolic[7] = NULL; m_addr_offset = 0; m_neg_pred = false; m_is_return_var = false; -- cgit v1.3 From a9af79ac84b69fa18dd395349b88f0d984f0a505 Mon Sep 17 00:00:00 2001 From: J Date: Tue, 7 Aug 2018 15:12:52 -0700 Subject: working fix for deadlock due to operand collector, parser changes to support culaunchkernel --- libcuda/cuda_runtime_api.cc | 74 ++++++--------------------------------------- src/cuda-sim/ptx_ir.h | 12 ++++++++ src/cuda-sim/ptx_parser.cc | 34 +++++++++++++++++++++ src/gpgpu-sim/shader.cc | 6 ++++ 4 files changed, 62 insertions(+), 64 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index c87c6c3..c12aaeb 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -3033,7 +3033,6 @@ CUresult CUDAAPI cuModuleGetFunction(CUfunction *hfunc, CUmodule hmod, const cha function_info* f = symtab->lookup_function( std::string(name) ); //just need to add given pointer to map for cudaLaunch context->register_hostFun_function( (const char*) hfunc, f); - g_cuda_launch_stack.push_back( kernel_config() ); *hfunc = (CUfunction)hfunc; return CUDA_SUCCESS; } @@ -3047,60 +3046,6 @@ CUresult CUDAAPI cuModuleUnload(CUmodule hmod) return CUDA_SUCCESS; } -CUresult CUDAAPI cuFuncSetBlockShape(CUfunction hfunc, int x, int y, int z) -{ - if(g_debug_execution >= 3){ - announce_call(__my_func__); - } - gpgpusim_ptx_assert( !g_cuda_launch_stack.empty(), "empty launch stack" ); - kernel_config &config = g_cuda_launch_stack.back(); - dim3 *d = new dim3(x,y,z); - config.set_block_dim(d); - - return CUDA_SUCCESS; -} - -CUresult CUDAAPI cuParamSetSize(CUfunction hfunc, unsigned int numbytes) -{ - if(g_debug_execution >= 3){ - announce_call(__my_func__); - } - //check if size matches given args - gpgpusim_ptx_assert( !g_cuda_launch_stack.empty(), "empty launch stack" ); - kernel_config &config = g_cuda_launch_stack.back(); - gpgpu_ptx_sim_arg_list_t args = config.get_args(); - size_t total_size = 0; - for( gpgpu_ptx_sim_arg_list_t::iterator a = args.begin(); a != args.end(); a++ ) { - total_size += a->m_nbytes; - } - return (numbytes==total_size) ? CUDA_SUCCESS : CUDA_ERROR_INVALID_VALUE; -} - -CUresult CUDAAPI cuParamSetv(CUfunction hfunc, int offset, void *ptr, unsigned int numbytes) -{ - if(g_debug_execution >= 3){ - announce_call(__my_func__); - } - cudaSetupArgument((const void *) ptr, (size_t) numbytes, (size_t) offset); - return CUDA_SUCCESS; -} - -CUresult CUDAAPI cuLaunchGrid(CUfunction f, int grid_width, int grid_height) -{ - if(g_debug_execution >= 3){ - announce_call(__my_func__); - } - const char *hostFun = (const char*) f; - gpgpusim_ptx_assert( !g_cuda_launch_stack.empty(), "empty launch stack" ); - kernel_config &config = g_cuda_launch_stack.back(); - dim3 *d = new dim3(grid_width,grid_height,1); - config.set_grid_dim(d); - - cudaLaunch(hostFun); - return CUDA_SUCCESS; -} - - CUresult CUDAAPI cuLaunchKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void **kernelParams, void **extra) @@ -3108,17 +3053,18 @@ CUresult CUDAAPI cuLaunchKernel(CUfunction f, unsigned int gridDimX, unsigned in if(g_debug_execution >= 3){ announce_call(__my_func__); } - if (sharedMemBytes!=0||hStream!=NULL||kernelParams!=NULL||extra!=NULL){ - printf("GPGPU-Sim CUDA DRIVER API: Warning: Currently do not support \nsharedMemBytes, hStream, kernelParams, and extra parameters.\n"); + if (extra!=NULL){ + printf("GPGPU-Sim CUDA DRIVER API: ERROR: Currently do not support void** extra.\n"); + abort(); } const char *hostFun = (const char*) f; - gpgpusim_ptx_assert( !g_cuda_launch_stack.empty(), "empty launch stack" ); - kernel_config &config = g_cuda_launch_stack.back(); - dim3 *d_b = new dim3(blockDimX, blockDimY, blockDimZ); - config.set_block_dim(d_b); - dim3 *d_g = new dim3(gridDimX, gridDimY, gridDimZ); - config.set_grid_dim(d_g); - + CUctx_st *context = GPGPUSim_Context(); + function_info *entry = context->get_kernel(hostFun); + cudaConfigureCall(dim3(gridDimX, gridDimY, gridDimZ), dim3(blockDimX, blockDimY, blockDimZ), sharedMemBytes, (cudaStream_t) hStream); + for(unsigned i = 0; i < entry->num_args(); i++){ + std::pair p = entry->get_param_config(i); + cudaSetupArgument(kernelParams[i], p.first, p.second); + } cudaLaunch(hostFun); return CUDA_SUCCESS; } diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index ef4cf48..e726ab9 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1293,6 +1293,17 @@ public: bool is_pdom_set() const { return pdom_done; } //return pdom flag void set_pdom() { pdom_done = true; } //set pdom flag + void add_config_param( size_t size, unsigned alignment ){ + unsigned offset = 0; + if (m_param_configs.size()>0){ + unsigned offset_nom = m_param_configs.back().first + m_param_configs.back().second; + offset = offset_nom%alignment ? (offset_nom/alignment + 1) * alignment : offset_nom; + } + m_param_configs.push_back(std::pair(size, offset)); + } + + std::pair get_param_config(unsigned param_num) const { return m_param_configs[param_num]; } + private: unsigned m_uid; unsigned m_local_mem_framesize; @@ -1306,6 +1317,7 @@ private: unsigned m_instr_mem_size; std::map m_kernel_params; std::map m_ptx_kernel_param_info; + std::vector< std::pair > m_param_configs; const symbol *m_return_var_sym; std::vector m_args; std::list m_instructions; diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index c418fac..e6d6325 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -61,6 +61,7 @@ memory_space_t g_ptr_spec = undefined_space; int g_scalar_type_spec = -1; int g_vector_spec = -1; int g_alignment_spec = -1; +int g_size = -1; int g_extern_spec = 0; // variable declaration stuff: @@ -116,6 +117,7 @@ void init_directive_state() g_vector_spec=-1; g_opcode=-1; g_alignment_spec = -1; + g_size = -1; g_extern_spec = 0; g_scalar_type.clear(); g_operands.clear(); @@ -373,6 +375,9 @@ int pad_address (new_addr_type address, unsigned size, unsigned maxalign) { void add_identifier( const char *identifier, int array_dim, unsigned array_ident ) { + if(array_ident==ARRAY_IDENTIFIER){ + g_size *= array_dim; + } if( g_func_decl && (g_func_info == NULL) ) { // return variable decl... assert( g_add_identifier_cached__identifier == NULL ); @@ -562,10 +567,13 @@ void add_constptr(const char* identifier1, const char* identifier2, int offset) void add_function_arg() { + assert(g_size>0); if( g_func_info ) { PTX_PARSE_DPRINTF("add_function_arg \"%s\"", g_last_symbol->name().c_str() ); g_func_info->add_arg(g_last_symbol); + g_func_info->add_config_param( g_size, g_alignment_spec ); } + } void add_extern_spec() @@ -617,6 +625,32 @@ void add_vector_spec(int spec ) void add_scalar_type_spec( int type_spec ) { + //save size of parameter + switch ( type_spec ) { + case B8_TYPE: + case S8_TYPE: + case U8_TYPE: + g_size = 1; break; + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: + case F16_TYPE: + g_size = 2; break; + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: + case F32_TYPE: + g_size = 4; break; + case B64_TYPE: + case BB64_TYPE: + case S64_TYPE: + case U64_TYPE: + case F64_TYPE: + case FF64_TYPE: + g_size = 8; break; + case BB128_TYPE: + g_size = 16; break; + } PTX_PARSE_DPRINTF("add_scalar_type_spec \"%s\"", g_ptx_token_decode[type_spec].c_str()); g_scalar_type.push_back( type_spec ); if ( g_scalar_type.size() > 1 ) { diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index 4640d65..da85bae 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -3034,6 +3034,12 @@ bool opndcoll_rfu_t::writeback( const warp_inst_t &inst ) for( r=regs.begin(); r!=regs.end();r++,n++ ) { unsigned reg = *r; unsigned bank = register_bank(reg,inst.warp_id(),m_num_banks,m_bank_warp_shift); + unsigned count = 0; + while( !m_arbiter.bank_idle(bank) ) { + assert((++count) Date: Tue, 7 Aug 2018 18:53:26 -0700 Subject: implemented prmt and started working on variable precision mul inst --- cuda-kernels/v4p_kernel.cu | 232 +++++++++++++++++++++++++++++++++++++++++++ src/cuda-sim/instructions.cc | 91 ++++++++++++++++- src/cuda-sim/ptx.l | 7 +- src/cuda-sim/ptx.y | 16 ++- src/cuda-sim/ptx_ir.cc | 25 +++-- src/cuda-sim/ptx_ir.h | 2 + src/cuda-sim/ptx_parser.cc | 2 +- 7 files changed, 362 insertions(+), 13 deletions(-) create mode 100644 cuda-kernels/v4p_kernel.cu (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/cuda-kernels/v4p_kernel.cu b/cuda-kernels/v4p_kernel.cu new file mode 100644 index 0000000..bb9064b --- /dev/null +++ b/cuda-kernels/v4p_kernel.cu @@ -0,0 +1,232 @@ +#include +#include + +// Define some error checking macros. +#define cudaErrCheck(stat) { cudaErrCheck_((stat), __FILE__, __LINE__); } +void cudaErrCheck_(cudaError_t stat, const char *file, int line) { + if (stat != cudaSuccess) { + fprintf(stderr, "CUDA Error: %s %s %d\n", cudaGetErrorString(stat), file, line); + } +} + +#define curandErrCheck(stat) { curandErrCheck_((stat), __FILE__, __LINE__); } +void curandErrCheck_(curandStatus_t stat, const char *file, int line) { + if (stat != CURAND_STATUS_SUCCESS) { + fprintf(stderr, "cuRand Error: %d %s %d\n", stat, file, line); + } +} + +#include +using namespace nvcuda; + +// Must be multiples of 16 for wmma code to work +#define MATRIX_M (16) +#define MATRIX_N (16) +#define MATRIX_K (16) + + +// The only dimensions currently supported by WMMA +const int WMMA_M = 16; +const int WMMA_N = 16; +const int WMMA_K = 16; + +__global__ void wmma_example(half *a, half *b, float *c,float *d_fp16, int M, int N, int K) { + //unsigned int start_time=0,end_time=0; + //start_time=clock(); + + // Declare the fragments + wmma::fragment a_frag; + wmma::fragment b_frag; + wmma::fragment c_frag; + + // Bounds checking + wmma::load_matrix_sync(a_frag, a, K); + wmma::load_matrix_sync(b_frag, b, K); + wmma::load_matrix_sync(c_frag, c, N,wmma::mem_col_major); + wmma::mma_sync(c_frag, a_frag, b_frag, c_frag); + + wmma::store_matrix_sync(d_fp16, c_frag, N, wmma::mem_col_major); + //printf("clock=%d",end_time-start_time); +} + +__global__ void convertFp32ToFp16 (half *out, float *in, int n) { + int idx = blockDim.x * blockIdx.x + threadIdx.x; + if (idx < n) { + out[idx] = in[idx]; + } +} +__global__ void convertFp16ToFp32 (float *out, half *in, int n) { + int idx = blockDim.x * blockIdx.x + threadIdx.x; + if (idx < n) { + out[idx] = in[idx]; + } +} + +__global__ void convertInt32ToInt8 (int *out, int *in, int n) { + int idx = blockDim.x * blockIdx.x + threadIdx.x; + if (idx < n/4) { + out[idx] =(in[4*idx]&0xff)|(in[4*idx+1]&0xff)<<8|(in[4*idx+2]&0xff)<<16|(in[4*idx+3]&0xff)<<24; + } +} + +__global__ void convertInt8ToInt32 (int *out, int *in, int n) { + int idx = blockDim.x * blockIdx.x + threadIdx.x; + int shft_amt=8*(idx%4); + int shft_mask=0xff<>shft_amt; + } +} + +int main(int argc, char* argv[]) { + int *a_int32; + int *b_int32; + int *c_int32; + int *d_int32; + + int *a_int8; + int *b_int8; + + int *a_host_wmma; + int *b_host_wmma; + int *c_host_wmma; + int *d_host_wmma; + int *d_cal_host_wmma; + + cudaEvent_t startWMMA; + cudaEvent_t stopWMMA; + + + cudaErrCheck(cudaEventCreate(&startWMMA)); + cudaErrCheck(cudaEventCreate(&stopWMMA)); + + // Use tensor cores + cudaErrCheck(cudaMalloc((void**)&a_int32, MATRIX_M * MATRIX_K * sizeof(int))); + cudaErrCheck(cudaMalloc((void**)&b_int32, MATRIX_K * MATRIX_N * sizeof(int))); + cudaErrCheck(cudaMalloc((void**)&c_int32, MATRIX_K * MATRIX_N * sizeof(int))); + cudaErrCheck(cudaMalloc((void**)&d_int32, MATRIX_K * MATRIX_N * sizeof(int))); + cudaErrCheck(cudaMalloc((void**)&a_int8, MATRIX_M * MATRIX_K * sizeof(int)/4)); + cudaErrCheck(cudaMalloc((void**)&b_int8, MATRIX_K * MATRIX_N * sizeof(int)/4)); + + + a_host_wmma = (int *)malloc(MATRIX_M * MATRIX_K * sizeof(int)); + b_host_wmma = (int *)malloc(MATRIX_K * MATRIX_N * sizeof(int)); + c_host_wmma = (int *)malloc(MATRIX_M * MATRIX_N * sizeof(int)); + d_host_wmma = (int *)malloc(MATRIX_M * MATRIX_N * sizeof(int)); + d_cal_host_wmma = (int *)malloc(MATRIX_M * MATRIX_N * sizeof(int)); + + printf("a_int32\n"); + for(int m=0;m>> (a_int8, a_int32, MATRIX_M * MATRIX_K); + convertInt8ToInt32 <<< (MATRIX_M * MATRIX_K + 255) / 256, 256 >>> (d_int32, a_int8, MATRIX_M * MATRIX_K); + //convertFp32ToFp16 <<< (MATRIX_K * MATRIX_N + 255) / 256, 256 >>> (b_fp16, b_fp32, MATRIX_K * MATRIX_N); + //convertFp32ToFp16 <<< (MATRIX_M * MATRIX_N + 255) / 256, 256 >>> (c_fp16, c_fp32, MATRIX_K * MATRIX_N); + cudaErrCheck(cudaMemcpy(d_host_wmma, d_int32, MATRIX_M * MATRIX_N * sizeof(float), cudaMemcpyDeviceToHost)); + + +//AAMIR printf("\nM = %d, N = %d, K = %d. \n", MATRIX_M, MATRIX_N, MATRIX_K); +//AAMIR +//AAMIR printf("Running with wmma...\n"); +//AAMIR cudaErrCheck(cudaEventRecord(startWMMA)); +//AAMIR wmma_example <<< 1, 32>>> (a_fp16, b_fp16, c_fp32, d_fp32 , MATRIX_M, MATRIX_N, MATRIX_K); +//AAMIR cudaErrCheck(cudaEventRecord(stopWMMA)); +//AAMIR cudaErrCheck(cudaEventSynchronize(stopWMMA)); +//AAMIR +//AAMIR // Error checking +//AAMIR printf("\nChecking results...\n"); +//AAMIR cudaErrCheck(cudaMemcpy(d_host_wmma, d_fp32, MATRIX_M * MATRIX_N * sizeof(float), cudaMemcpyDeviceToHost)); +//AAMIR +//AAMIR printf("Results verified: cublas and WMMA agree.\n\n"); +//AAMIR float wmmaTime; +//AAMIR cudaErrCheck(cudaEventElapsedTime(&wmmaTime, startWMMA, stopWMMA)); +//AAMIR printf("wmma took %fms\n", wmmaTime); +//AAMIR +//AAMIR cudaErrCheck(cudaEventDestroy(startWMMA)); +//AAMIR cudaErrCheck(cudaEventDestroy(stopWMMA)); +//AAMIR +//AAMIR int t=200000; +//AAMIR while(t-->0); +//AAMIR printf("D_CALCULATED\n"); +//AAMIR +//AAMIR for(int m=0;m1) +//AAMIR { +//AAMIR printf("ERROR:\n"); +//AAMIR suc=0; +//AAMIR } +//AAMIR } +//AAMIR } +//AAMIR if(suc==1) +//AAMIR printf("COMPLETED_SUCCESSFULLY\n"); +//AAMIR + + cudaErrCheck(cudaFree(a_int32)); + cudaErrCheck(cudaFree(b_int32)); + cudaErrCheck(cudaFree(c_int32)); + cudaErrCheck(cudaFree(d_int32)); + cudaErrCheck(cudaFree(a_int8)); + cudaErrCheck(cudaFree(b_int8)); + + free(a_host_wmma); + free(b_host_wmma); + free(c_host_wmma); + free(d_host_wmma); + cudaErrCheck(cudaDeviceReset()); + return 0; +} + + diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 7af157f..aaee2a2 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -48,7 +48,7 @@ using half_float::half; unsigned ptx_instruction::g_num_ptx_inst_uid=0; -bool g_debug_instruction = 1; +bool g_debug_instruction = 0; const char *g_opcode_string[NUM_OPCODES] = { @@ -3911,7 +3911,94 @@ void popc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) } void prefetch_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } void prefetchu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void prmt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } + +int prmt_mode_present(int mode) +{ + int returnval=0; + switch(mode){ + case PRMT_F4E_MODE: + case PRMT_B4E_MODE: + case PRMT_RC8_MODE: + case PRMT_RC16_MODE: + case PRMT_ECL_MODE: + case PRMT_ECR_MODE: + returnval=1; + break; + default: + break; + } + return returnval; +} +int read_byte(int mode,int control,int d_sel_index,signed long long value){ + + int returnval; + int prmt_f4e_mode[4][4]={{0,1,2,3},{1,2,3,4},{2,3,4,5},{3,4,5,6}}; + int prmt_b4e_mode[4][4]={{0,7,6,5},{1,0,7,6},{2,1,0,7},{3,2,1,0}}; + int prmt_rc8_mode[4][4]={{0,0,0,0},{1,1,1,1},{2,2,2,2},{3,3,3,3}}; + int prmt_ecl_mode[4][4]={{0,1,2,3},{1,1,2,3},{2,2,2,3},{3,3,3,3}}; + int prmt_ecr_mode[4][4]={{0,0,0,0},{0,1,1,1},{0,1,2,2},{0,1,2,3}}; + int prmt_rc16_mode[4][4]={{0,1,0,1},{2,3,2,3},{0,1,0,1},{2,3,2,3}}; + + if(!prmt_mode_present(mode)){ + if(control&0x8){ + returnval=0xff; + } + else{ + returnval= (value>>(8*control)) & 0xff; + } + } + else{ + switch(mode){ + case PRMT_F4E_MODE: returnval=prmt_f4e_mode[control][d_sel_index];break; + case PRMT_B4E_MODE: returnval=prmt_b4e_mode[control][d_sel_index];break; + case PRMT_RC8_MODE: returnval=prmt_rc8_mode[control][d_sel_index];break; + case PRMT_ECL_MODE: returnval=prmt_ecl_mode[control][d_sel_index];break; + case PRMT_ECR_MODE: returnval=prmt_ecr_mode[control][d_sel_index];break; + case PRMT_RC16_MODE: returnval=prmt_rc16_mode[control][d_sel_index];break; + default: printf("ERROR\n");break; + } + } + return (returnval<<8*d_sel_index); +} + +void prmt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { + + ptx_reg_t src1_data, src2_data, src3_data,tmpdata,data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + unsigned mode = pI->prmt_op(); + unsigned i_type = pI->get_type(); + + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + + tmpdata.s64=src1_data.s32|(src2_data.s64<<32); + int ctl[4]; + + if(!prmt_mode_present(mode)){ + ctl[0]=(src3_data.s32>>0)&0xf; + ctl[1]=(src3_data.s32>>4)&0xf; + ctl[2]=(src3_data.s32>>8)&0xf; + ctl[3]=(src3_data.s32>>12)&0xf; + } + else{ + ctl[0]=ctl[1]=ctl[2]=ctl[3]=(src3_data.s32>>0)&0x3; + } + + data.s32=0; + data.s32=data.s32|read_byte(mode,ctl[0],0,tmpdata.s64); //First byte-0 + data.s32=data.s32|read_byte(mode,ctl[1],1,tmpdata.s64); //Second byte-1 + data.s32=data.s32|read_byte(mode,ctl[2],2,tmpdata.s64); //Third byte-2 + data.s32=data.s32|read_byte(mode,ctl[3],3,tmpdata.s64); //Fourth byte-3 + + thread->set_operand_value(dst,data, i_type, thread, pI); + + +} void rcp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { diff --git a/src/cuda-sim/ptx.l b/src/cuda-sim/ptx.l index 03d6838..a6b6fcc 100644 --- a/src/cuda-sim/ptx.l +++ b/src/cuda-sim/ptx.l @@ -164,7 +164,12 @@ breakaddr TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE; \.row TC; ptx_lval.int_value = ROW; return LAYOUT; \.col TC; ptx_lval.int_value = COL; return LAYOUT; \.m16n16k16 TC; ptx_lval.int_value = M16N16K16; return CONFIGURATION; - +\.f4e TC; return PRMT_F4E_MODE; +\.b4e TC; return PRMT_B4E_MODE; +\.rc8 TC; return PRMT_RC8_MODE; +\.ecl TC; return PRMT_ECL_MODE; +\.ecr TC; return PRMT_ECR_MODE; +\.rc16 TC; return PRMT_RC16_MODE; \.align TC; return ALIGN_DIRECTIVE; \.branchtargets TC; return BRANCHTARGETS_DIRECTIVE; diff --git a/src/cuda-sim/ptx.y b/src/cuda-sim/ptx.y index 8744663..cd455dd 100644 --- a/src/cuda-sim/ptx.y +++ b/src/cuda-sim/ptx.y @@ -50,7 +50,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %token PTR_DIRECTIVE %token ENTRY_DIRECTIVE %token EXTERN_DIRECTIVE -%token WEAK_DIRECTIVE %token FILE_DIRECTIVE %token FUNC_DIRECTIVE %token GLOBAL_DIRECTIVE @@ -203,6 +202,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %token DOWN_OPTION; %token BFLY_OPTION; %token IDX_OPTION; +%token PRMT_F4E_MODE; +%token PRMT_B4E_MODE; +%token PRMT_RC8_MODE; +%token PRMT_RC16_MODE; +%token PRMT_ECL_MODE; +%token PRMT_ECR_MODE; %type function_decl_header %type function_decl @@ -432,6 +437,7 @@ option: type_spec | addressable_spec | rounding_mode | wmma_spec + | prmt_spec | SYNC_OPTION { add_option(SYNC_OPTION); } | ARRIVE_OPTION { add_option(ARRIVE_OPTION); } | RED_OPTION { add_option(RED_OPTION); } @@ -520,6 +526,14 @@ compare_spec:EQ_OPTION { add_option(EQ_OPTION); } | NAN_OPTION { add_option(NAN_OPTION); } ; +prmt_spec: PRMT_F4E_MODE { add_option( PRMT_F4E_MODE); } + | PRMT_B4E_MODE { add_option( PRMT_B4E_MODE); } + | PRMT_RC8_MODE { add_option( PRMT_RC8_MODE); } + | PRMT_RC16_MODE{ add_option( PRMT_RC16_MODE);} + | PRMT_ECL_MODE { add_option( PRMT_ECL_MODE); } + | PRMT_ECR_MODE { add_option( PRMT_ECR_MODE); } + ; + wmma_spec: WMMA_DIRECTIVE LAYOUT CONFIGURATION{add_space_spec(global_space,0);add_ptr_spec(global_space); add_wmma_option($1);add_wmma_option($2);add_wmma_option($3);} | WMMA_DIRECTIVE LAYOUT LAYOUT CONFIGURATION{add_wmma_option($1);add_wmma_option($2);add_wmma_option($3);add_wmma_option($4);} ; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index 9c2ac69..d12c741 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -1233,16 +1233,25 @@ ptx_instruction::ptx_instruction( int opcode, case HALF_OPTION: m_inst_size = 4; // bytes break; - case EXTP_OPTION: - break; - case NC_OPTION: - break; - case UP_OPTION: - case DOWN_OPTION: - case BFLY_OPTION: - case IDX_OPTION: + case EXTP_OPTION: + break; + case NC_OPTION: + break; + case UP_OPTION: + case DOWN_OPTION: + case BFLY_OPTION: + case IDX_OPTION: m_shfl_op = last_ptx_inst_option; break; + + case PRMT_F4E_MODE: + case PRMT_B4E_MODE: + case PRMT_RC8_MODE: + case PRMT_ECL_MODE: + case PRMT_ECR_MODE: + case PRMT_RC16_MODE: + m_prmt_op = last_ptx_inst_option; + break; default: assert(0); break; diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index cff312b..cb4556e 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1086,6 +1086,7 @@ public: unsigned dimension() const { return m_geom_spec;} unsigned barrier_op() const {return m_barrier_op;} unsigned shfl_op() const {return m_shfl_op;} + unsigned prmt_op() const {return m_prmt_op;} enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot }; enum vote_mode_t vote_mode() const { return m_vote_mode; } @@ -1156,6 +1157,7 @@ private: unsigned m_saturation_mode; unsigned m_barrier_op; unsigned m_shfl_op; + unsigned m_prmt_op; std::list m_scalar_type; memory_space_t m_space_spec; diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index eb81961..6757091 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -39,7 +39,7 @@ void set_ptx_warp_size(const struct core_config * warp_size) g_shader_core_config=warp_size; } -static bool g_debug_ir_generation=true; +static bool g_debug_ir_generation=false; const char *g_filename; unsigned g_max_regs_per_thread = 0; -- cgit v1.3 From cbe8e3ddbca3895a499882c31b080a58a7545d25 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Thu, 16 Aug 2018 14:10:55 -0700 Subject: fix alignment bug in parser, modified ptxjitconfig, minor fixes --- src/cuda-sim/cuda-sim.cc | 45 ++++++++++++++++++++++++++++++++++++++++++-- src/cuda-sim/instructions.cc | 8 ++++---- src/cuda-sim/ptx_ir.h | 1 + src/cuda-sim/ptx_parser.cc | 2 +- 4 files changed, 49 insertions(+), 7 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 5b80616..b85ba95 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1232,7 +1232,6 @@ void function_info::ptx_jit_config(std::map mallocPt std::vector< std::pair > param_data; std::vector offsets; std::vector paramIsPointer; - char buff[1024]; char * gpgpusim_path = getenv("GPGPUSIM_ROOT"); assert(gpgpusim_path!=NULL); @@ -1241,6 +1240,47 @@ void function_info::ptx_jit_config(std::map mallocPt std::string command = std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; std::string filename(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/params.config" + std::to_string(counter)); + //initialize paramList + char buff[1024]; + std::string filename_c(filename+"_c"); + snprintf(buff,1024,"c++filt %s > %s", get_name().c_str(), filename_c.c_str()); + system(buff); + FILE *fp = fopen(filename_c.c_str(), "r"); + fgets(buff, 1024, fp); + fclose(fp); + std::string fn(buff); + size_t pos1, pos2; + pos1 = fn.find_last_of("("); + pos2 = fn.find(")", pos1); + assert(pos2>pos1&&pos1>0); + strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); + char *tok; + tok = strtok(buff, ","); + std::string tmp; + while(tok!=NULL){ + std::string param(tok); + if(param.find("<")!=std::string::npos){ + assert(param.find(">")==std::string::npos); + assert(param.find("*")==std::string::npos); + tmp = param; + } else { + if (tmp.length()>0){ + tmp = ""; + assert(param.find(">")!=std::string::npos); + assert(param.find("<")==std::string::npos); + assert(param.find("*")==std::string::npos); + } + printf("%s\n", param.c_str()); + if(param.find("*")!=std::string::npos){ + paramIsPointer.push_back(true); + }else{ + paramIsPointer.push_back(false); + } + } + tok = strtok(NULL, ","); + } + + for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { param_info &p = i->second; std::string name = p.get_name(); @@ -1249,8 +1289,9 @@ void function_info::ptx_jit_config(std::map mallocPt param_t param_value = p.get_value(); offsets.push_back((unsigned)p.get_offset()); - if(param_value.size==sizeof(void*) && mallocPtr_Size.find(*(unsigned long long*)param_value.pdata)!=mallocPtr_Size.end()){ + if (paramIsPointer[i->first] && (*(unsigned long long*)param_value.pdata != 0)){ //is pointer + assert(param_value.size==sizeof(void*)&&"MisID'd this param as pointer"); size_t array_size = 0; unsigned long long param_pointer = *(unsigned long long*)param_value.pdata; if(mallocPtr_Size.find(param_pointer)!=mallocPtr_Size.end()){ diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 109cfa7..31fc434 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -1471,17 +1471,17 @@ void brev_impl( const ptx_instruction *pI, ptx_thread_info *thread ) switch(i_type){ case B32_TYPE: msb = 31; - for (unsigned i=0; i<=msb; i++) { + for (unsigned i=0; i<=msb; i++) { if((src1_data.u32 & (1 << i))) data.u32 |= 1 << (msb - i); - } + } break; case B64_TYPE: msb = 63; - for (unsigned i=0; i<=msb; i++) { + for (unsigned i=0; i<=msb; i++) { if((src1_data.u64 & (1 << i))) data.u64 |= 1 << (msb - i); - } + } break; default: assert(0); } diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index e726ab9..768b3e7 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1297,6 +1297,7 @@ public: unsigned offset = 0; if (m_param_configs.size()>0){ unsigned offset_nom = m_param_configs.back().first + m_param_configs.back().second; + //ensure offset matches alignment requirements offset = offset_nom%alignment ? (offset_nom/alignment + 1) * alignment : offset_nom; } m_param_configs.push_back(std::pair(size, offset)); diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index 3f3485c..0ba9a5b 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -572,7 +572,7 @@ void add_function_arg() PTX_PARSE_DPRINTF("add_function_arg \"%s\"", g_last_symbol->name().c_str() ); g_func_info->add_arg(g_last_symbol); unsigned alignment = (g_alignment_spec==-1) ? g_size : g_alignment_spec; - assert(alignment<=8); + assert(alignment==1||alignment==2||alignment==4||alignment==8||alignment==16);//known valid alignment values g_func_info->add_config_param( g_size, alignment); } -- cgit v1.3 From 9e7cd8867d76fb99eadfadfa09947ff057d012d3 Mon Sep 17 00:00:00 2001 From: negargoli93 Date: Thu, 16 Aug 2018 15:20:07 -0700 Subject: Timing model for VCORE --- src/abstract_hardware_model.h | 2 ++ src/cuda-sim/cuda-sim.cc | 28 ++++++++++++++++--- src/cuda-sim/ptx_ir.h | 4 +-- src/gpgpu-sim/gpu-sim.cc | 16 +++++++++-- src/gpgpu-sim/shader.cc | 62 ++++++++++++++++++++++++++++++++++++++++--- src/gpgpu-sim/shader.h | 57 ++++++++++++++++++++++++++++++++++----- 6 files changed, 152 insertions(+), 17 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index 7fe5d82..781509f 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -78,6 +78,7 @@ enum uarch_op_t { ALU_OP=1, SFU_OP, TENSOR_CORE_OP, + VP_CORE_OP, ALU_SFU_OP, LOAD_OP, STORE_OP, @@ -135,6 +136,7 @@ enum operation_pipeline_t { SP__OP, SFU__OP, TENSOR_CORE__OP, + VP_CORE__OP, MEM__OP }; typedef enum operation_pipeline_t operation_pipeline; diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 2fe5667..1ad12ee 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -542,7 +542,7 @@ void ptx_instruction::set_mul_div_or_other_archop(){ sp_op=INT_DIV_OP; break; default: - if((op==ALU_OP)||(op==TENSOR_CORE_OP)) + if((op==ALU_OP)||(op==VP_CORE_OP)) sp_op=INT__OP; break; } @@ -649,9 +649,11 @@ void ptx_instruction::set_opcode_and_latency() break; case LD_OP: op = LOAD_OP; break; case MMA_LD_OP: op = LOAD_OP; break; + case VP_LD_OP: op = LOAD_OP; break; case LDU_OP: op = LOAD_OP; break; case ST_OP: op = STORE_OP; break; case MMA_ST_OP: op = STORE_OP; break; + case VP_ST_OP: op = STORE_OP; break; case BRA_OP: op = BRANCH_OP; break; case BREAKADDR_OP: op = BRANCH_OP; break; case TEX_OP: op = LOAD_OP; mem_op=TEX; break; @@ -799,6 +801,26 @@ void ptx_instruction::set_opcode_and_latency() initiation_interval = 64; op=TENSOR_CORE_OP; break; + case VP_MMA_OP: + if(get_wmma_type()==VP_MMA4) + { + latency = 5; + initiation_interval = 5; + } + if(get_wmma_type()==VP_MMA8) + { + latency = 5; + initiation_interval = 5; + } + if(get_wmma_type()==VP_MMA16) + { + latency = 5; + initiation_interval = 5; + } + op=VP_CORE_OP; + op=VP_CORE_OP; + op=VP_CORE_OP; + break; case SHFL_OP: latency = 32; initiation_interval = 15; @@ -900,10 +922,10 @@ void ptx_instruction::pre_decode() case WT_OPTION: cache_op = CACHE_WRITE_THROUGH; break; default: //if( m_opcode == LD_OP || m_opcode == LDU_OP ) - if( m_opcode == MMA_LD_OP || m_opcode == LD_OP || m_opcode == LDU_OP ) + if( m_opcode ==VP_LD_OP || m_opcode == MMA_LD_OP || m_opcode == LD_OP || m_opcode == LDU_OP ) cache_op = CACHE_ALL; //else if( m_opcode == ST_OP ) - else if( m_opcode == ST_OP || m_opcode == ST_OP ) + else if( m_opcode == VP_ST_OP ||m_opcode == MMA_ST_OP || m_opcode == ST_OP ) cache_op = CACHE_WRITE_BACK; else if( m_opcode == ATOM_OP ) cache_op = CACHE_GLOBAL; diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index cb4556e..e025013 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1093,7 +1093,7 @@ public: int membar_level() const { return m_membar_level; } bool has_memory_read() const { - if( m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP|| m_opcode==MMA_LD_OP) + if( m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP|| m_opcode==MMA_LD_OP || m_opcode==VP_LD_OP) return true; // Check PTXPlus operand type below // Source operands are memory operands @@ -1105,7 +1105,7 @@ public: return false; } bool has_memory_write() const { - if( m_opcode == ST_OP || m_opcode==MMA_ST_OP ) return true; + if( m_opcode == ST_OP || m_opcode==MMA_ST_OP || m_opcode==VP_ST_OP ) return true; // Check PTXPlus operand type below // Destination operand is a memory operand ptx_instruction::const_iterator op=op_iter_begin(); diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 3e064c7..7a797b5 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -309,6 +309,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_operand_collector_num_units_tensor_core", OPT_INT32, &gpgpu_operand_collector_num_units_tensor_core, "number of collector units (default = 4)", "4"); + option_parser_register(opp, "-gpgpu_operand_collector_num_units_vp_core", OPT_INT32, &gpgpu_operand_collector_num_units_vp_core, + "number of collector units (default = 4)", + "4"); option_parser_register(opp, "-gpgpu_operand_collector_num_units_mem", OPT_INT32, &gpgpu_operand_collector_num_units_mem, "number of collector units (default = 2)", "2"); @@ -324,6 +327,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_tensor_core", OPT_INT32, &gpgpu_operand_collector_num_in_ports_tensor_core, "number of collector unit in ports (default = 1)", "1"); + option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_vp_core", OPT_INT32, &gpgpu_operand_collector_num_in_ports_vp_core, + "number of collector unit in ports (default = 1)", + "1"); option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_mem", OPT_INT32, &gpgpu_operand_collector_num_in_ports_mem, "number of collector unit in ports (default = 1)", "1"); @@ -339,6 +345,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_tensor_core", OPT_INT32, &gpgpu_operand_collector_num_out_ports_tensor_core, "number of collector unit in ports (default = 1)", "1"); + option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_vp_core", OPT_INT32, &gpgpu_operand_collector_num_out_ports_vp_core, + "number of collector unit in ports (default = 1)", + "1"); option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_mem", OPT_INT32, &gpgpu_operand_collector_num_out_ports_mem, "number of collector unit in ports (default = 1)", "1"); @@ -359,8 +368,8 @@ void shader_core_config::reg_options(class OptionParser * opp) "1"); option_parser_register(opp, "-gpgpu_pipeline_widths", OPT_CSTR, &pipeline_widths_string, "Pipeline widths " - "ID_OC_SP,ID_OC_SFU,ID_OC_TENSOR_CORE,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_TENSOR_CORE,OC_EX_MEM,EX_WB", - "1,1,1,1,1,1,1,1,1" ); + "ID_OC_SP,ID_OC_SFU,ID_OC_TENSOR_CORE,ID_OC_VP_CORE,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_TENSOR_CORE,OC_EX_TENSOR_CORE,OC_EX_MEM,EX_WB", + "1,1,1,1,1,1,1,1,1,1,1" ); option_parser_register(opp, "-gpgpu_num_sp_units", OPT_INT32, &gpgpu_num_sp_units, "Number of SP units (default=1)", "1"); @@ -370,6 +379,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_num_tensor_core_units", OPT_INT32, &gpgpu_num_tensor_core_units, "Number of tensor_core units (default=1)", "1"); + option_parser_register(opp, "-gpgpu_num_vp_core_units", OPT_INT32, &gpgpu_num_vp_core_units, + "Number of vp_core units (default=1)", + "1"); option_parser_register(opp, "-gpgpu_num_mem_units", OPT_INT32, &gpgpu_num_mem_units, "Number if ldst units (default=1) WARNING: not hooked up to anything", "1"); diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index 226e7f0..6f11ad9 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -149,6 +149,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_TENSOR_CORE], + &m_pipeline_reg[ID_OC_VP_CORE], &m_pipeline_reg[ID_OC_MEM], i ) @@ -164,6 +165,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_TENSOR_CORE], + &m_pipeline_reg[ID_OC_VP_CORE], &m_pipeline_reg[ID_OC_MEM], i, config->gpgpu_scheduler_string @@ -180,6 +182,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_TENSOR_CORE], + &m_pipeline_reg[ID_OC_VP_CORE], &m_pipeline_reg[ID_OC_MEM], i ) @@ -195,6 +198,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_TENSOR_CORE], + &m_pipeline_reg[ID_OC_VP_CORE], &m_pipeline_reg[ID_OC_MEM], i, config->gpgpu_scheduler_string @@ -215,10 +219,11 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, } //op collector configuration - enum { SP_CUS, SFU_CUS, TENSOR_CORE_CUS, MEM_CUS, GEN_CUS }; + enum { SP_CUS, SFU_CUS, TENSOR_CORE_CUS, VP_CORE_CUS, MEM_CUS, GEN_CUS }; m_operand_collector.add_cu_set(SP_CUS, m_config->gpgpu_operand_collector_num_units_sp, m_config->gpgpu_operand_collector_num_out_ports_sp); m_operand_collector.add_cu_set(SFU_CUS, m_config->gpgpu_operand_collector_num_units_sfu, m_config->gpgpu_operand_collector_num_out_ports_sfu); m_operand_collector.add_cu_set(TENSOR_CORE_CUS, config->gpgpu_operand_collector_num_units_tensor_core, config->gpgpu_operand_collector_num_out_ports_tensor_core); + m_operand_collector.add_cu_set(VP_CORE_CUS, config->gpgpu_operand_collector_num_units_vp_core, config->gpgpu_operand_collector_num_out_ports_vp_core); m_operand_collector.add_cu_set(MEM_CUS, m_config->gpgpu_operand_collector_num_units_mem, m_config->gpgpu_operand_collector_num_out_ports_mem); m_operand_collector.add_cu_set(GEN_CUS, m_config->gpgpu_operand_collector_num_units_gen, m_config->gpgpu_operand_collector_num_out_ports_gen); @@ -252,6 +257,14 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, in_ports.clear(),out_ports.clear(),cu_sets.clear(); } + for (unsigned i = 0; i < config->gpgpu_operand_collector_num_in_ports_vp_core; i++) { + in_ports.push_back(&m_pipeline_reg[ID_OC_VP_CORE]); + out_ports.push_back(&m_pipeline_reg[OC_EX_VP_CORE]); + cu_sets.push_back((unsigned)VP_CORE_CUS); + cu_sets.push_back((unsigned)GEN_CUS); + m_operand_collector.add_port(in_ports,out_ports,cu_sets); + in_ports.clear(),out_ports.clear(),cu_sets.clear(); + } for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_mem; i++) { in_ports.push_back(&m_pipeline_reg[ID_OC_MEM]); @@ -267,10 +280,12 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, in_ports.push_back(&m_pipeline_reg[ID_OC_SP]); in_ports.push_back(&m_pipeline_reg[ID_OC_SFU]); in_ports.push_back(&m_pipeline_reg[ID_OC_TENSOR_CORE]); + in_ports.push_back(&m_pipeline_reg[ID_OC_VP_CORE]); in_ports.push_back(&m_pipeline_reg[ID_OC_MEM]); out_ports.push_back(&m_pipeline_reg[OC_EX_SP]); out_ports.push_back(&m_pipeline_reg[OC_EX_SFU]); out_ports.push_back(&m_pipeline_reg[OC_EX_TENSOR_CORE]); + out_ports.push_back(&m_pipeline_reg[OC_EX_VP_CORE]); out_ports.push_back(&m_pipeline_reg[OC_EX_MEM]); cu_sets.push_back((unsigned)GEN_CUS); m_operand_collector.add_port(in_ports,out_ports,cu_sets); @@ -280,7 +295,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_operand_collector.init( m_config->gpgpu_num_reg_banks, this ); // execute - m_num_function_units = m_config->gpgpu_num_sp_units + m_config->gpgpu_num_sfu_units + config->gpgpu_num_tensor_core_units + 1; // sp_unit, sfu, ldst_unit + m_num_function_units = m_config->gpgpu_num_sp_units + m_config->gpgpu_num_sfu_units + config->gpgpu_num_tensor_core_units + config->gpgpu_num_vp_core_units + 1; // sp_unit, sfu, ldst_unit //m_dispatch_port = new enum pipeline_stage_name_t[ m_num_function_units ]; //m_issue_port = new enum pipeline_stage_name_t[ m_num_function_units ]; @@ -304,6 +319,12 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_issue_port.push_back(OC_EX_TENSOR_CORE); } + for (int k = 0; k < config->gpgpu_num_vp_core_units; k++) { + m_fu.push_back(new vp_core( &m_pipeline_reg[EX_WB], m_config, this )); + m_dispatch_port.push_back(ID_OC_VP_CORE); + m_issue_port.push_back(OC_EX_VP_CORE); + } + m_ldst_unit = new ldst_unit( m_icnt, m_mem_fetch_allocator, this, &m_operand_collector, m_scoreboard, config, mem_config, stats, shader_id, tpc_id ); m_fu.push_back(m_ldst_unit); m_dispatch_port.push_back(ID_OC_MEM); @@ -910,7 +931,8 @@ void scheduler_unit::cycle() bool sp_pipe_avail = m_sp_out->has_free(); bool sfu_pipe_avail = m_sfu_out->has_free(); bool tensor_core_pipe_avail = m_tensor_core_out->has_free(); - if( sp_pipe_avail && (pI->op != SFU_OP) && (pI->op != TENSOR_CORE_OP)) { + bool vp_core_pipe_avail = m_vp_core_out->has_free(); + if( sp_pipe_avail && (pI->op != SFU_OP) && (pI->op != TENSOR_CORE_OP) && (pI->op !=VP_CORE_OP)) { //Jin: special for CDP api if(pI->m_is_cdp && !warp(warp_id).m_cdp_dummy) { @@ -950,6 +972,14 @@ void scheduler_unit::cycle() issued_inst=true; warp_inst_issued = true; } + } + else if ( (pI->op == VP_CORE_OP) ) { + if( vp_core_pipe_avail ) { + m_shader->issue_warp(*m_vp_core_out,pI,active_mask,warp_id); + issued++; + issued_inst=true; + warp_inst_issued = true; + } } } } else { SCHED_DPRINTF( "Warp (warp_id %u, dynamic_warp_id %u) fails scoreboard\n", @@ -1116,10 +1146,11 @@ swl_scheduler::swl_scheduler ( shader_core_stats* stats, shader_core_ctx* shader register_set* sp_out, register_set* sfu_out, register_set* tensor_core_out, + register_set* vp_core_out, register_set* mem_out, int id, char* config_string ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, tensor_core_out, mem_out, id ) + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, tensor_core_out, vp_core_out, mem_out, id ) { unsigned m_prioritization_readin; int ret = sscanf( config_string, @@ -1548,6 +1579,12 @@ tensor_core:: tensor_core( register_set* result_port, const shader_core_config m_name = "TENSOR_CORE"; } +vp_core:: vp_core( register_set* result_port, const shader_core_config *config,shader_core_ctx *core ) + : pipelined_simd_unit(result_port,config,config->max_vp_core_latency,core) +{ + m_name = "VP_CORE"; +} + void sfu::issue( register_set& source_reg ) { @@ -1569,6 +1606,16 @@ void tensor_core::issue( register_set& source_reg ) pipelined_simd_unit::issue(source_reg); } +void vp_core::issue( register_set& source_reg ) +{ + warp_inst_t** ready_reg = source_reg.get_ready(); + //m_core->incexecstat((*ready_reg)); + + (*ready_reg)->op_pipe= VP_CORE__OP; + m_core->incsfu_stat(m_core->get_config()->warp_size,(*ready_reg)->latency); + pipelined_simd_unit::issue(source_reg); +} + void ldst_unit::active_lanes_in_pipeline(){ unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline(); @@ -1599,6 +1646,13 @@ void tensor_core::active_lanes_in_pipeline(){ m_core->incfumemactivelanes_stat(active_count); } +void vp_core::active_lanes_in_pipeline(){ + unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline(); + assert(active_count<=m_core->get_config()->warp_size); + m_core->incsfuactivelanes_stat(active_count); + m_core->incfuactivelanes_stat(active_count); + m_core->incfumemactivelanes_stat(active_count); +} sp_unit::sp_unit( register_set* result_port, const shader_core_config *config,shader_core_ctx *core) : pipelined_simd_unit(result_port,config,config->max_sp_latency,core) diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index 90a3134..d292d56 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -319,11 +319,12 @@ public: register_set* sp_out, register_set* sfu_out, register_set* tensor_core_out, + register_set* vp_core_out, register_set* mem_out, int id) : m_supervised_warps(), m_stats(stats), m_shader(shader), m_scoreboard(scoreboard), m_simt_stack(simt), /*m_pipeline_reg(pipe_regs),*/ m_warp(warp), - m_sp_out(sp_out),m_sfu_out(sfu_out),m_tensor_core_out(tensor_core_out),m_mem_out(mem_out), m_id(id){} + m_sp_out(sp_out),m_sfu_out(sfu_out),m_tensor_core_out(tensor_core_out),m_vp_core_out(vp_core_out),m_mem_out(mem_out), m_id(id){} virtual ~scheduler_unit(){} virtual void add_supervised_warp_id(int i) { m_supervised_warps.push_back(&warp(i)); @@ -397,6 +398,7 @@ protected: register_set* m_sp_out; register_set* m_sfu_out; register_set* m_tensor_core_out; + register_set* m_vp_core_out; register_set* m_mem_out; int m_id; @@ -410,9 +412,10 @@ public: register_set* sp_out, register_set* sfu_out, register_set* tensor_core_out, + register_set* vp_core_out, register_set* mem_out, int id ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, tensor_core_out, mem_out, id ){} + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, tensor_core_out, vp_core_out, mem_out, id ){} virtual ~lrr_scheduler () {} virtual void order_warps (); virtual void done_adding_supervised_warps() { @@ -428,9 +431,10 @@ public: register_set* sp_out, register_set* sfu_out, register_set* tensor_core_out, + register_set* vp_core_out, register_set* mem_out, int id ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, tensor_core_out, mem_out, id ){} + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, tensor_core_out,vp_core_out, mem_out, id ){} virtual ~gto_scheduler () {} virtual void order_warps (); virtual void done_adding_supervised_warps() { @@ -448,10 +452,11 @@ public: register_set* sp_out, register_set* sfu_out, register_set* tensor_core_out, + register_set* vp_core_out, register_set* mem_out, int id, char* config_str ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, tensor_core_out, mem_out, id ), + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, tensor_core_out, vp_core_out, mem_out, id ), m_pending_warps() { unsigned inner_level_readin; @@ -499,6 +504,7 @@ public: register_set* sp_out, register_set* sfu_out, register_set* tensor_core_out, + register_set* vp_core_out, register_set* mem_out, int id, char* config_string ); @@ -1083,6 +1089,22 @@ public: virtual void active_lanes_in_pipeline(); virtual void issue( register_set& source_reg ); }; +class vp_core : public pipelined_simd_unit +{ +public: + vp_core( register_set* result_port, const shader_core_config *config, shader_core_ctx *core ); + virtual bool can_issue( const warp_inst_t &inst ) const + { + switch(inst.op) { + case VP_CORE_OP: break; + default: return false; + } + return pipelined_simd_unit::can_issue(inst); + } + virtual void active_lanes_in_pipeline(); + virtual void issue( register_set& source_reg ); +}; + class sp_unit : public pipelined_simd_unit @@ -1225,10 +1247,12 @@ enum pipeline_stage_name_t { ID_OC_SP=0, ID_OC_SFU, ID_OC_TENSOR_CORE, + ID_OC_VP_CORE, ID_OC_MEM, OC_EX_SP, OC_EX_SFU, OC_EX_TENSOR_CORE, + OC_EX_VP_CORE, OC_EX_MEM, EX_WB, N_PIPELINE_STAGES @@ -1238,10 +1262,12 @@ const char* const pipeline_stage_name_decode[] = { "ID_OC_SP", "ID_OC_SFU", "ID_OC_TENSOR_CORE", + "ID_OC_VP_CORE", "ID_OC_MEM", "OC_EX_SP", "OC_EX_SFU", "OC_EX_TENSOR_CORE", + "OC_EX_VP_CORE", "OC_EX_MEM", "EX_WB", "N_PIPELINE_STAGES" @@ -1286,13 +1312,21 @@ struct shader_core_config : public core_config max_warps_per_shader = n_thread_per_shader/warp_size; assert( !(n_thread_per_shader % warp_size) ); max_sfu_latency = 512; - max_tensor_core_latency = 64; max_sp_latency = 32; + + max_tensor_core_latency = 64; gpgpu_num_tensor_core_units=8; gpgpu_operand_collector_num_units_tensor_core=24; gpgpu_operand_collector_num_in_ports_tensor_core=8; gpgpu_operand_collector_num_out_ports_tensor_core=8; - m_L1I_config.init(m_L1I_config.m_config_string,FuncCachePreferNone); + + max_vp_core_latency = 64; + gpgpu_num_vp_core_units=8; + gpgpu_operand_collector_num_units_vp_core=24; + gpgpu_operand_collector_num_in_ports_vp_core=8; + gpgpu_operand_collector_num_out_ports_vp_core=8; + + m_L1I_config.init(m_L1I_config.m_config_string,FuncCachePreferNone); m_L1T_config.init(m_L1T_config.m_config_string,FuncCachePreferNone); m_L1C_config.init(m_L1C_config.m_config_string,FuncCachePreferNone); m_L1D_config.init(m_L1D_config.m_config_string,FuncCachePreferNone); @@ -1339,24 +1373,28 @@ struct shader_core_config : public core_config int gpgpu_operand_collector_num_units_sp; int gpgpu_operand_collector_num_units_sfu; int gpgpu_operand_collector_num_units_tensor_core; + int gpgpu_operand_collector_num_units_vp_core; int gpgpu_operand_collector_num_units_mem; int gpgpu_operand_collector_num_units_gen; unsigned int gpgpu_operand_collector_num_in_ports_sp; unsigned int gpgpu_operand_collector_num_in_ports_sfu; unsigned int gpgpu_operand_collector_num_in_ports_tensor_core; + unsigned int gpgpu_operand_collector_num_in_ports_vp_core; unsigned int gpgpu_operand_collector_num_in_ports_mem; unsigned int gpgpu_operand_collector_num_in_ports_gen; unsigned int gpgpu_operand_collector_num_out_ports_sp; unsigned int gpgpu_operand_collector_num_out_ports_sfu; unsigned int gpgpu_operand_collector_num_out_ports_tensor_core; + unsigned int gpgpu_operand_collector_num_out_ports_vp_core; unsigned int gpgpu_operand_collector_num_out_ports_mem; unsigned int gpgpu_operand_collector_num_out_ports_gen; int gpgpu_num_sp_units; int gpgpu_num_sfu_units; int gpgpu_num_tensor_core_units; + int gpgpu_num_vp_core_units; int gpgpu_num_mem_units; //Shader core resources @@ -1370,6 +1408,7 @@ struct shader_core_config : public core_config unsigned max_sp_latency; unsigned max_sfu_latency; unsigned max_tensor_core_latency; + unsigned max_vp_core_latency; unsigned n_simt_cores_per_cluster; unsigned n_simt_clusters; @@ -1408,6 +1447,7 @@ struct shader_core_stats_pod { unsigned *m_num_sp_acesses; unsigned *m_num_sfu_acesses; unsigned *m_num_tensor_core_acesses; + unsigned *m_num_vp_core_acesses; unsigned *m_num_trans_acesses; unsigned *m_num_mem_acesses; unsigned *m_num_sp_committed; @@ -1415,6 +1455,7 @@ struct shader_core_stats_pod { unsigned *m_num_tlb_accesses; unsigned *m_num_sfu_committed; unsigned *m_num_tensor_core_committed; + unsigned *m_num_vp_core_committed; unsigned *m_num_mem_committed; unsigned *m_read_regfile_acesses; unsigned *m_write_regfile_acesses; @@ -1424,6 +1465,7 @@ struct shader_core_stats_pod { unsigned *m_active_sp_lanes; unsigned *m_active_sfu_lanes; unsigned *m_active_tensor_core_lanes; + unsigned *m_active_vp_core_lanes; unsigned *m_active_fu_lanes; unsigned *m_active_fu_mem_lanes; unsigned *m_n_diverge; // number of divergence occurring in this shader @@ -1496,6 +1538,7 @@ public: m_num_sp_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_num_sfu_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_num_tensor_core_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); + m_num_vp_core_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_num_trans_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_num_mem_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_num_sp_committed= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); @@ -1504,10 +1547,12 @@ public: m_active_sp_lanes= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_active_sfu_lanes= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_active_tensor_core_lanes= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); + m_active_vp_core_lanes= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_active_fu_lanes= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_active_fu_mem_lanes= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_num_sfu_committed= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_num_tensor_core_committed= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); + m_num_vp_core_committed= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_num_mem_committed= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_read_regfile_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_write_regfile_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); -- cgit v1.3 From eae030c9d1d607d1c14e4ade99cb5caea6403efd Mon Sep 17 00:00:00 2001 From: aamir Date: Sat, 3 Nov 2018 17:03:58 -0700 Subject: merged with memory subsytem. Regression is passing but tensorcore kernel is stuck in deadlock --- .gitignore | 1 + .travis.yml | 31 +- CHANGES | 30 + Jenkinsfile | 126 ++++ README | 10 +- aerialvision/organizedata.py | 2 +- configs/GTX480/config_fermi_islip.icnt | 70 -- configs/GTX480/gpgpusim.config | 134 ---- configs/GTX480/gpuwattch_gtx480.xml | 538 -------------- configs/GeForceGTX1080Ti/config_fermi_islip.icnt | 70 -- configs/GeForceGTX1080Ti/gpgpusim.config | 150 ---- configs/GeForceGTX1080Ti/gpuwattch_gtx1080Ti.xml | 538 -------------- configs/GeForceGTX750Ti/config_fermi_islip.icnt | 70 -- configs/GeForceGTX750Ti/gpgpusim.config | 131 ---- configs/GeForceGTX750Ti/gpuwattch_gtx750Ti.xml | 539 -------------- configs/Pascal_TITANX/config_fermi_islip.icnt | 70 -- configs/Pascal_TITANX/gpgpusim.config | 156 ---- configs/QuadroFX5600/gpgpusim.config | 99 --- configs/QuadroFX5600/gpuwattch_quadrofx5600.xml | 538 -------------- configs/QuadroFX5600/icnt_config_islip.icnt | 70 -- configs/QuadroFX5800/config_quadro_islip.icnt | 69 -- configs/QuadroFX5800/gpgpusim.config | 88 --- configs/TeslaC2050/config_fermi_islip.icnt | 70 -- configs/TeslaC2050/gpgpusim.config | 133 ---- .../deprecated-cfgs/GTX480/config_fermi_islip.icnt | 70 ++ configs/deprecated-cfgs/GTX480/gpgpusim.config | 134 ++++ .../deprecated-cfgs/GTX480/gpuwattch_gtx480.xml | 538 ++++++++++++++ .../GeForceGTX750Ti/config_fermi_islip.icnt | 70 ++ .../GeForceGTX750Ti/gpgpusim.config | 131 ++++ .../GeForceGTX750Ti/gpuwattch_gtx750Ti.xml | 539 ++++++++++++++ .../deprecated-cfgs/QuadroFX5600/gpgpusim.config | 99 +++ .../QuadroFX5600/gpuwattch_quadrofx5600.xml | 538 ++++++++++++++ .../QuadroFX5600/icnt_config_islip.icnt | 70 ++ .../QuadroFX5800/config_quadro_islip.icnt | 69 ++ .../deprecated-cfgs/QuadroFX5800/gpgpusim.config | 88 +++ .../SM6_GTX1080/config_fermi_islip.icnt | 70 ++ .../deprecated-cfgs/SM6_GTX1080/gpgpusim.config | 150 ++++ .../SM6_GTX1080/gpuwattch_gtx1080Ti.xml | 538 ++++++++++++++ .../SM6_P100/config_fermi_islip.icnt | 70 ++ configs/deprecated-cfgs/SM6_P100/gpgpusim.config | 156 ++++ .../deprecated-cfgs/SM6_P100/gpuwattch_gtx480.xml | 538 ++++++++++++++ .../SM6_TITANX/config_fermi_islip.icnt | 70 ++ configs/deprecated-cfgs/SM6_TITANX/gpgpusim.config | 157 +++++ .../SM6_TITANX/gpuwattch_gtx480.xml | 538 ++++++++++++++ .../SM7_TITANV/config_fermi_islip.icnt | 70 ++ configs/deprecated-cfgs/SM7_TITANV/gpgpusim.config | 165 +++++ .../TeslaC2050/config_fermi_islip.icnt | 70 ++ configs/deprecated-cfgs/TeslaC2050/gpgpusim.config | 133 ++++ .../tested-cfgs/SM2_GTX480/config_fermi_islip.icnt | 70 ++ configs/tested-cfgs/SM2_GTX480/gpgpusim.config | 153 ++++ .../tested-cfgs/SM2_GTX480/gpuwattch_gtx480.xml | 538 ++++++++++++++ .../tested-cfgs/SM6_TITANX/config_fermi_islip.icnt | 73 ++ configs/tested-cfgs/SM6_TITANX/gpgpusim.config | 185 +++++ .../tested-cfgs/SM7_TITANV/config_fermi_islip.icnt | 74 ++ configs/tested-cfgs/SM7_TITANV/gpgpusim.config | 191 +++++ cuobjdump_to_ptxplus/ptx_parser.h | 4 + libcuda/cuda_runtime_api.cc | 49 +- nightly.jenkinsfile | 75 ++ setup_environment | 2 +- src/abstract_hardware_model.cc | 49 +- src/abstract_hardware_model.h | 33 +- src/cuda-sim/cuda-sim.cc | 47 +- src/cuda-sim/instructions.cc | 1 + src/cuda-sim/ptx.y | 3 +- src/cuda-sim/ptx_ir.cc | 7 +- src/cuda-sim/ptx_ir.h | 7 + src/cuda-sim/ptx_parser.cc | 4 + src/cuda-sim/ptx_parser.h | 1 + src/gpgpu-sim/addrdec.cc | 71 ++ src/gpgpu-sim/addrdec.h | 9 + src/gpgpu-sim/delayqueue.h | 1 + src/gpgpu-sim/dram.cc | 637 +++++++++++++---- src/gpgpu-sim/dram.h | 77 +- src/gpgpu-sim/dram_sched.cc | 121 +++- src/gpgpu-sim/dram_sched.h | 12 + src/gpgpu-sim/gpu-cache.cc | 782 +++++++++++++++++---- src/gpgpu-sim/gpu-cache.h | 644 +++++++++++++++-- src/gpgpu-sim/gpu-sim.cc | 125 +++- src/gpgpu-sim/gpu-sim.h | 57 +- src/gpgpu-sim/l2cache.cc | 166 ++++- src/gpgpu-sim/l2cache.h | 18 + src/gpgpu-sim/l2cache_trace.h | 16 + src/gpgpu-sim/mem_fetch.cc | 7 +- src/gpgpu-sim/mem_fetch.h | 13 +- src/gpgpu-sim/mem_latency_stat.cc | 19 +- src/gpgpu-sim/mem_latency_stat.h | 4 + src/gpgpu-sim/shader.cc | 431 ++++++++++-- src/gpgpu-sim/shader.h | 93 ++- src/gpgpusim_entrypoint.cc | 1 + src/gpuwattch/makefile | 4 +- src/intersim2/Makefile | 2 +- src/intersim2/interconnect_interface.cpp | 9 +- src/trace_streams.tup | 2 + version | 2 +- 94 files changed, 9589 insertions(+), 4104 deletions(-) create mode 100644 Jenkinsfile delete mode 100644 configs/GTX480/config_fermi_islip.icnt delete mode 100644 configs/GTX480/gpgpusim.config delete mode 100755 configs/GTX480/gpuwattch_gtx480.xml delete mode 100644 configs/GeForceGTX1080Ti/config_fermi_islip.icnt delete mode 100644 configs/GeForceGTX1080Ti/gpgpusim.config delete mode 100755 configs/GeForceGTX1080Ti/gpuwattch_gtx1080Ti.xml delete mode 100644 configs/GeForceGTX750Ti/config_fermi_islip.icnt delete mode 100644 configs/GeForceGTX750Ti/gpgpusim.config delete mode 100755 configs/GeForceGTX750Ti/gpuwattch_gtx750Ti.xml delete mode 100644 configs/Pascal_TITANX/config_fermi_islip.icnt delete mode 100644 configs/Pascal_TITANX/gpgpusim.config delete mode 100644 configs/QuadroFX5600/gpgpusim.config delete mode 100644 configs/QuadroFX5600/gpuwattch_quadrofx5600.xml delete mode 100644 configs/QuadroFX5600/icnt_config_islip.icnt delete mode 100644 configs/QuadroFX5800/config_quadro_islip.icnt delete mode 100644 configs/QuadroFX5800/gpgpusim.config delete mode 100644 configs/TeslaC2050/config_fermi_islip.icnt delete mode 100644 configs/TeslaC2050/gpgpusim.config create mode 100644 configs/deprecated-cfgs/GTX480/config_fermi_islip.icnt create mode 100644 configs/deprecated-cfgs/GTX480/gpgpusim.config create mode 100755 configs/deprecated-cfgs/GTX480/gpuwattch_gtx480.xml create mode 100644 configs/deprecated-cfgs/GeForceGTX750Ti/config_fermi_islip.icnt create mode 100644 configs/deprecated-cfgs/GeForceGTX750Ti/gpgpusim.config create mode 100755 configs/deprecated-cfgs/GeForceGTX750Ti/gpuwattch_gtx750Ti.xml create mode 100644 configs/deprecated-cfgs/QuadroFX5600/gpgpusim.config create mode 100644 configs/deprecated-cfgs/QuadroFX5600/gpuwattch_quadrofx5600.xml create mode 100644 configs/deprecated-cfgs/QuadroFX5600/icnt_config_islip.icnt create mode 100644 configs/deprecated-cfgs/QuadroFX5800/config_quadro_islip.icnt create mode 100644 configs/deprecated-cfgs/QuadroFX5800/gpgpusim.config create mode 100644 configs/deprecated-cfgs/SM6_GTX1080/config_fermi_islip.icnt create mode 100644 configs/deprecated-cfgs/SM6_GTX1080/gpgpusim.config create mode 100755 configs/deprecated-cfgs/SM6_GTX1080/gpuwattch_gtx1080Ti.xml create mode 100644 configs/deprecated-cfgs/SM6_P100/config_fermi_islip.icnt create mode 100644 configs/deprecated-cfgs/SM6_P100/gpgpusim.config create mode 100755 configs/deprecated-cfgs/SM6_P100/gpuwattch_gtx480.xml create mode 100644 configs/deprecated-cfgs/SM6_TITANX/config_fermi_islip.icnt create mode 100644 configs/deprecated-cfgs/SM6_TITANX/gpgpusim.config create mode 100755 configs/deprecated-cfgs/SM6_TITANX/gpuwattch_gtx480.xml create mode 100644 configs/deprecated-cfgs/SM7_TITANV/config_fermi_islip.icnt create mode 100644 configs/deprecated-cfgs/SM7_TITANV/gpgpusim.config create mode 100644 configs/deprecated-cfgs/TeslaC2050/config_fermi_islip.icnt create mode 100644 configs/deprecated-cfgs/TeslaC2050/gpgpusim.config create mode 100644 configs/tested-cfgs/SM2_GTX480/config_fermi_islip.icnt create mode 100644 configs/tested-cfgs/SM2_GTX480/gpgpusim.config create mode 100755 configs/tested-cfgs/SM2_GTX480/gpuwattch_gtx480.xml create mode 100644 configs/tested-cfgs/SM6_TITANX/config_fermi_islip.icnt create mode 100644 configs/tested-cfgs/SM6_TITANX/gpgpusim.config create mode 100644 configs/tested-cfgs/SM7_TITANV/config_fermi_islip.icnt create mode 100644 configs/tested-cfgs/SM7_TITANV/gpgpusim.config create mode 100644 nightly.jenkinsfile (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/.gitignore b/.gitignore index d21cc8f..6c6ed5f 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,5 @@ cuobjdump_to_ptxplus/sass_parser.hh cuobjdump_to_ptxplus/sass_parser.output build/* +tags *.swp diff --git a/.travis.yml b/.travis.yml index 7a12a1c..b38f468 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ sudo: required before_install: - - docker pull tgrogers/gpgpu-sim_regress:latest + - docker pull tgrogers/gpgpu-sim_regress:volta_update language: cpp @@ -10,34 +10,13 @@ matrix: include: - services: docker env: - - CONFIG=configs.gtx480.yml + - CONFIG=GTX480 - CUDA_INSTALL_PATH=/usr/local/cuda-4.2/ - PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-4.2/ -# This config is just taking far too long... -# - services: docker -# env: -# - CONFIG=configs.gtx750ti.yml -# - CUDA_INSTALL_PATH=/usr/local/cuda-4.2/ -# - PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-4.2/ - services: docker env: - - CONFIG=configs.quadro5600.yml - - CUDA_INSTALL_PATH=/usr/local/cuda-4.2/ - - PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-4.2/ - - services: docker - env: - - CONFIG=configs.quadro5800.yml - - CUDA_INSTALL_PATH=/usr/local/cuda-4.2/ - - PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-4.2/ - - services: docker - env: - - CONFIG=configs.teslac2050.yml - - CUDA_INSTALL_PATH=/usr/local/cuda-4.2/ - - PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-4.2/ - - services: docker - env: - - CONFIG=configs.gtx1080ti.yml - - CUDA_INSTALL_PATH=/usr/local/cuda-4.2/ + - CONFIG=TITANV + - CUDA_INSTALL_PATH=/usr/local/cuda-9.1/ - PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-9.1/ -script: docker run -v `pwd`:/home/runner/gpgpu-sim_distribution:rw tgrogers/gpgpu-sim_regress:latest /bin/bash -c "./start_torque.sh; chown -R runner /home/runner/gpgpu-sim_distribution; su - runner -c 'export CUDA_INSTALL_PATH=$CUDA_INSTALL_PATH && export PTXAS_CUDA_INSTALL_PATH=$PTXAS_CUDA_INSTALL_PATH && source /home/runner/gpgpu-sim_distribution/setup_environment && make -j -C /home/runner/gpgpu-sim_distribution && cd /home/runner/gpgpu-sim_simulations/ && git pull && /home/runner/gpgpu-sim_simulations/util/job_launching/run_simulations.py -c /home/runner/gpgpu-sim_simulations/util/job_launching/regression_recipies/rodinia_2.0-ft/$CONFIG -N regress && /home/runner/gpgpu-sim_simulations/util/job_launching/monitor_func_test.py -v -N regress'" +script: docker run -v `pwd`:/home/runner/gpgpu-sim_distribution:rw tgrogers/gpgpu-sim_regress:volta_update /bin/bash -c "./start_torque.sh; chown -R runner /home/runner/gpgpu-sim_distribution; su - runner -c 'export CUDA_INSTALL_PATH=$CUDA_INSTALL_PATH && export PTXAS_CUDA_INSTALL_PATH=$PTXAS_CUDA_INSTALL_PATH && source /home/runner/gpgpu-sim_distribution/setup_environment && make -j -C /home/runner/gpgpu-sim_distribution && cd /home/runner/gpgpu-sim_simulations/ && git pull && /home/runner/gpgpu-sim_simulations/util/job_launching/run_simulations.py -C $CONFIG -B rodinia_2.0-ft -N regress && /home/runner/gpgpu-sim_simulations/util/job_launching/monitor_func_test.py -v -N regress'" diff --git a/CHANGES b/CHANGES index dbf5b39..60be885 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,34 @@ LOG: +Version 4.0.0 (development branch) versus 3.2.3 +-Front-End: +1- Support .nc cache modifier and __ldg function to access the read-only L1D cache +2- Partially-support some SASS_60 in the PTXP_PLUS (not completed yet) +-GPU Core: +1- Fermi/Pascal coalescer: coalescing on 32-bytes sectors. +2- Adding separate dp unit pipeline. +3- diff dual issue: allow scheduler to issue diff insts at a time +4- Fair memory issue from multiple schedulers. +-Cache System: +1- Sector L1/L2 cache +2- Fetch-on-write and lazy-fetch-on-read write allocation policy. +3- Improving the L1 cache throughput (streaming L1 cache) +4- Performance model for CUDA memory copy. +5- Support memory partition indexing to reduce partition camping (POLY, XOR and PAE (ISCA’18) Indexing) +6- Adaptive cache configuration +-Memory: +1- Performance Model for HBM (mainly the dual-bus interface) +2- Separate Read/Write buffers. +3- Advanced bank indexing function. +-Statistics: +1- Adding more detailed cache statistics to define and analyze cache bottlenecks. +2- Adding more detailed memory statistics (BLP, RBL, etc) to define and analyze memory bottlenecks. +3- Addig new system stats: gpu occupancy, L2BW, etc +-Configs: +Adding the Pascal and Volta config files that has been correlated against real hardware. +See the correlation website here: +https://engineering.purdue.edu/tgrogers/group/correlator.html + + Version 3.2.3+edits (development branch) versus 3.2.3 - Support for running regression tests using Travis - Support added for CUDA dynamic parallelism (courtesy of Jin Wang from Georgia Tech) diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..1969aea --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,126 @@ +pipeline { + agent { + label "purdue-cluster" + } + + options { + disableConcurrentBuilds() + } + + stages { + stage('simulator-build') { + steps { + parallel "4.2": { + sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/4.2_env_setup.sh &&\ + source `pwd`/setup_environment &&\ + make -j' + }, "9.1" : { + sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/9.1_env_setup.sh &&\ + source `pwd`/setup_environment &&\ + make -j' + } + } + } + stage('simulations-build'){ + steps{ + sh 'rm -rf gpgpu-sim_simulations' + sh 'git clone git@github.rcac.purdue.edu:TimRogersGroup/gpgpu-sim_simulations.git && \ + cd gpgpu-sim_simulations && \ + git checkout purdue-cluster && \ + git pull && \ + ln -s /home/tgrogers-raid/a/common/data_dirs benchmarks/' + sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/4.2_env_setup.sh &&\ + source `pwd`/setup_environment &&\ + cd gpgpu-sim_simulations && \ + source ./benchmarks/src/setup_environment && \ + make -j -C ./benchmarks/src rodinia_2.0-ft sdk-4.2 && \ + make -C ./benchmarks/src data' + sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/9.1_env_setup.sh &&\ + source `pwd`/setup_environment &&\ + cd gpgpu-sim_simulations && \ + source ./benchmarks/src/setup_environment && \ + make -j -C ./benchmarks/src/ rodinia_2.0-ft sdk-4.2 && \ + make -C ./benchmarks/src data' + } + } + stage('regress'){ + steps { + parallel "4.2-rodinia": { + sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/4.2_env_setup.sh &&\ + source `pwd`/setup_environment &&\ + ./gpgpu-sim_simulations/util/job_launching/run_simulations.py -B rodinia_2.0-ft -C GTX480,GTX480-PTXPLUS -N regress-$$ && \ + PLOTDIR="jenkins/${JOB_NAME}/${BUILD_NUMBER}/4.2-rodinia" && ssh tgrogers@dynamo.ecn.purdue.edu mkdir -p /home/dynamo/a/tgrogers/website/gpgpu-sim-plots/$PLOTDIR && \ + ./gpgpu-sim_simulations/util/job_launching/monitor_func_test.py -v -N regress-$$ -s stats-per-app-4.2.csv && \ + ./gpgpu-sim_simulations/util/plotting/plot-get-stats.py -c stats-per-app-4.2.csv -p tgrogers@dynamo.ecn.purdue.edu:~/website/gpgpu-sim-plots/$PLOTDIR -w https://engineering.purdue.edu/tgrogers/gpgpu-sim-plots/$PLOTDIR -n $PLOTDIR ' + }, "9.1-functest": { + sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/9.1_env_setup.sh &&\ + source `pwd`/setup_environment &&\ + ./gpgpu-sim_simulations/util/job_launching/run_simulations.py -B rodinia_2.0-ft,sdk-4.2 -C TITANX,TITANX-L1ON -N regress-$$ && \ + PLOTDIR="jenkins/${JOB_NAME}/${BUILD_NUMBER}/9.1-rodinia" && ssh tgrogers@dynamo.ecn.purdue.edu mkdir -p /home/dynamo/a/tgrogers/website/gpgpu-sim-plots/$PLOTDIR && \ + ./gpgpu-sim_simulations/util/job_launching/monitor_func_test.py -v -s stats-per-app-9.1.csv -N regress-$$ && \ + ./gpgpu-sim_simulations/util/plotting/plot-get-stats.py -c stats-per-app-9.1.csv -p tgrogers@dynamo.ecn.purdue.edu:~/website/gpgpu-sim-plots/$PLOTDIR -w https://engineering.purdue.edu/tgrogers/gpgpu-sim-plots/$PLOTDIR -n $PLOTDIR' + } + } + } + stage('4.2-correlate'){ + steps { + sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/4.2_env_setup.sh &&\ + source `pwd`/setup_environment &&\ + PLOTDIR="jenkins/${JOB_NAME}" &&\ + ./gpgpu-sim_simulations/util/job_launching/get_stats.py -R -K -k -B rodinia_2.0-ft -C GTX480,GTX480-PTXPLUS > stats-per-kernel-4.2.csv &&\ + ./gpgpu-sim_simulations/util/plotting/correlate_and_publish.sh stats-per-kernel-4.2.csv $PLOTDIR ${BUILD_NUMBER}' + } + } + stage('9.1-correlate'){ + steps { + sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/9.1_env_setup.sh &&\ + source `pwd`/setup_environment &&\ + PLOTDIR="jenkins/${JOB_NAME}" &&\ + ./gpgpu-sim_simulations/util/job_launching/get_stats.py -R -K -k -B rodinia_2.0-ft,sdk-4.2 -C TITANX,TITANX-L1ON > stats-per-kernel-9.1.csv &&\ + ./gpgpu-sim_simulations/util/plotting/correlate_and_publish.sh stats-per-kernel-9.1.csv $PLOTDIR ${BUILD_NUMBER}' + } + } + stage('archive-and-delta') { + steps { + sh 'rm -rf gpgpu-sim-results-repo' + sh 'git clone git@github.com:purdue-aalp/gpgpu-sim-results-repo.git' + sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/4.2_env_setup.sh &&\ + ./gpgpu-sim_simulations/util/job_launching/get_stats.py -R -K -k -B rodinia_2.0-ft -C GTX480-PTXPLUS > stats-per-kernel-4.2-ptxplus.csv &&\ + ./gpgpu-sim_simulations/util/job_launching/get_stats.py -R -K -k -B rodinia_2.0-ft -C GTX480 > stats-per-kernel-4.2-ptx.csv' + sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/9.1_env_setup.sh &&\ + ./gpgpu-sim_simulations/util/job_launching/get_stats.py -R -K -k -B rodinia_2.0-ft,sdk-4.2 -C TITANX > stats-per-kernel-9.1-titanx.csv' + sh './gpgpu-sim_simulations/util/plotting/merge-stats.py -c ./gpgpu-sim-results-repo/jenkins/quick-regress/AALP/gpgpu-sim_distribution/dev-purdue-integration/stats-per-app-4.2.csv,./stats-per-app-4.2.csv -R > per-app-merge-4.2.csv' + sh './gpgpu-sim_simulations/util/plotting/merge-stats.py -c ./gpgpu-sim-results-repo/jenkins/quick-regress/AALP/gpgpu-sim_distribution/dev-purdue-integration/stats-per-app-9.1.csv,./stats-per-app-9.1.csv -R > per-app-merge-9.1.csv' + sh 'PLOTDIR="jenkins/${JOB_NAME}" &&\ + ./gpgpu-sim_simulations/util/plotting/plot-get-stats.py -c per-app-merge-4.2.csv -p tgrogers@dynamo.ecn.purdue.edu:~/website/gpgpu-sim-plots/$PLOTDIR/deltas -w https://engineering.purdue.edu/tgrogers/gpgpu-sim-plots/$PLOTDIR -n $PLOTDIR/deltas &&\ + ./gpgpu-sim_simulations/util/plotting/plot-get-stats.py -c per-app-merge-9.1.csv -p tgrogers@dynamo.ecn.purdue.edu:~/website/gpgpu-sim-plots/$PLOTDIR/deltas -w https://engineering.purdue.edu/tgrogers/gpgpu-sim-plots/$PLOTDIR/deltas -n $PLOTDIR/deltas &&\ + ./gpgpu-sim_simulations/util/plotting/merge-stats.py -c ./gpgpu-sim-results-repo/jenkins/quick-regress/AALP/gpgpu-sim_distribution/dev-purdue-integration/stats-per-kernel-4.2-ptx.csv,./stats-per-kernel-4.2-ptx.csv -R > per-kernel-merge-4.2-ptx.csv &&\ + ./gpgpu-sim_simulations/util/plotting/merge-stats.py -c ./gpgpu-sim-results-repo/jenkins/quick-regress/AALP/gpgpu-sim_distribution/dev-purdue-integration/stats-per-kernel-4.2-ptxplus.csv,./stats-per-kernel-4.2-ptxplus.csv -R > per-kernel-merge-4.2-ptxplus.csv &&\ + ./gpgpu-sim_simulations/util/plotting/merge-stats.py -c ./gpgpu-sim-results-repo/jenkins/quick-regress/AALP/gpgpu-sim_distribution/dev-purdue-integration/stats-per-kernel-9.1-titanx.csv,./stats-per-kernel-9.1-titanx.csv -R > per-kernel-merge-9.1-titanx.csv &&\ + ./gpgpu-sim_simulations/util/plotting/correlate_and_publish.sh per-kernel-merge-4.2-ptx.csv $PLOTDIR ${BUILD_NUMBER} &&\ + ./gpgpu-sim_simulations/util/plotting/correlate_and_publish.sh per-kernel-merge-4.2-ptxplus.csv $PLOTDIR ${BUILD_NUMBER} &&\ + ./gpgpu-sim_simulations/util/plotting/correlate_and_publish.sh per-kernel-merge-9.1-titanx.csv $PLOTDIR ${BUILD_NUMBER} &&\ + mkdir -p ./jenkins/quick-regress/${JOB_NAME}/ && cp stats-per-*.csv ./jenkins/quick-regress/${JOB_NAME}/ &&\ + cd ./gpgpu-sim-results-repo &&\ + git diff --quiet && git diff --staged --quiet || git commit -am "Jenkins automated checkin ${BUILD_NUMBER}" &&\ + git push' + } + } + } + post { + success { + emailext body: "See ${BUILD_URL}", + recipientProviders: [[$class: 'CulpritsRecipientProvider'], + [$class: 'RequesterRecipientProvider']], + subject: "[AALP Jenkins] Build #${BUILD_NUMBER} - Success!", + to: 'tgrogers@purdue.edu' + } + failure { + emailext body: "See ${BUILD_URL}", + recipientProviders: [[$class: 'CulpritsRecipientProvider'], + [$class: 'RequesterRecipientProvider']], + subject: "[AALP Jenkins] Build #${BUILD_NUMBER} - ${currentBuild.result}", + to: 'tgrogers@purdue.edu' + } + } +} diff --git a/README b/README index 6e2d734..7eaae5d 100644 --- a/README +++ b/README @@ -5,8 +5,8 @@ AerialVision and a configurable and extensible energy model called GPUWattch. GPGPU-Sim and GPUWattch have been rigorously validated with performance and power measurements of real hardware GPUs. -This version of GPGPU-Sim has been tested with CUDA version 2.3, 3.1, 4.0, -5.0, 5.5, 6.0 and 7.5. +This version of GPGPU-Sim has been tested with CUDA version 4.2, +5.0, 5.5, 6.0 and 7.5, 8.0, 9.0, 9.1 Please see the copyright notice in the file COPYRIGHT distributed with this release in the same directory as this file. @@ -18,6 +18,12 @@ Analyzing CUDA Workloads Using a Detailed GPU Simulator, in IEEE International Symposium on Performance Analysis of Systems and Software (ISPASS), Boston, MA, April 19-21, 2009. +If you use the memory system in GPGPU-Sim, or the Volta/Pascal models, +please cite: +Mahmoud Khairy, Jain Akshay, Tor Aamodt, Timothy G Rogers, +Exploring Modern GPU Memory System Design Challenges through Accurate Modeling, arXiv:1810.07269, +https://arxiv.org/abs/1810.07269 + If you use the GPUWattch energy model in your research, please cite: Jingwen Leng, Tayler Hetherington, Ahmed ElTantawy, Syed Gilani, Nam Sung Kim, diff --git a/aerialvision/organizedata.py b/aerialvision/organizedata.py index 090b90f..ea947cd 100644 --- a/aerialvision/organizedata.py +++ b/aerialvision/organizedata.py @@ -97,7 +97,7 @@ def organizedata(fileVars): 'sparse':OrganizeSparse, # Vector data with 2D index (used by DRAM access stats) 'custom':0 } - data_type_char = {int:'I', float:'f'} + data_type_char = {int:'L', float:'d'} print "Organizing data into internal format..." diff --git a/configs/GTX480/config_fermi_islip.icnt b/configs/GTX480/config_fermi_islip.icnt deleted file mode 100644 index 7820e4e..0000000 --- a/configs/GTX480/config_fermi_islip.icnt +++ /dev/null @@ -1,70 +0,0 @@ -//21*1 fly with 32 flits per packet under gpgpusim injection mode -use_map = 0; -flit_size = 32; - -// currently we do not use this, see subnets below -network_count = 2; - -// Topology -topology = fly; -k = 27; -n = 1; - -// Routing - -routing_function = dest_tag; - -// Flow control - -num_vcs = 1; -vc_buf_size = 8; - -wait_for_tail_credit = 0; - -// Router architecture - -vc_allocator = islip; //separable_input_first; -sw_allocator = islip; //separable_input_first; -alloc_iters = 1; - -credit_delay = 0; -routing_delay = 0; -vc_alloc_delay = 1; -sw_alloc_delay = 1; - -input_speedup = 2; -output_speedup = 1; -internal_speedup = 1.0; - -// Traffic, GPGPU-Sim does not use this - -traffic = uniform; -packet_size ={{1,2,3,4},{10,20}}; -packet_size_rate={{1,1,1,1},{2,1}}; - -// Simulation - Don't change - -sim_type = gpgpusim; -//sim_type = latency; -injection_rate = 0.1; - -subnets = 2; - -// Always use read and write no matter following line -//use_read_write = 1; - - -read_request_subnet = 0; -read_reply_subnet = 1; -write_request_subnet = 0; -write_reply_subnet = 1; - -read_request_begin_vc = 0; -read_request_end_vc = 0; -write_request_begin_vc = 0; -write_request_end_vc = 0; -read_reply_begin_vc = 0; -read_reply_end_vc = 0; -write_reply_begin_vc = 0; -write_reply_end_vc = 0; - diff --git a/configs/GTX480/gpgpusim.config b/configs/GTX480/gpgpusim.config deleted file mode 100644 index ee90c12..0000000 --- a/configs/GTX480/gpgpusim.config +++ /dev/null @@ -1,134 +0,0 @@ -# functional simulator specification --gpgpu_ptx_instruction_classification 0 --gpgpu_ptx_sim_mode 0 --gpgpu_ptx_force_max_capability 20 - - -# SASS execution (only supported with CUDA >= 4.0) --gpgpu_ptx_convert_to_ptxplus 0 --gpgpu_ptx_save_converted_ptxplus 0 - -# high level architecture configuration --gpgpu_n_clusters 15 --gpgpu_n_cores_per_cluster 1 --gpgpu_n_mem 6 --gpgpu_n_sub_partition_per_mchannel 2 - -# Fermi clock domains -#-gpgpu_clock_domains ::: -# In Fermi, each pipeline has 16 execution units, so the Core clock needs to be divided -# by 2. (GPGPU-Sim simulates a warp (32 threads) in a single cycle). 1400/2 = 700 --gpgpu_clock_domains 700.0:700.0:700.0:924.0 - -# shader core pipeline config --gpgpu_shader_registers 32768 --gpgpu_occupancy_sm_number 20 - -# This implies a maximum of 48 warps/SM --gpgpu_shader_core_pipeline 1536:32 --gpgpu_shader_cta 8 --gpgpu_simd_model 1 - -# Pipeline widths and number of FUs -# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB --gpgpu_pipeline_widths 2,1,1,2,1,1,2 --gpgpu_num_sp_units 2 --gpgpu_num_sfu_units 1 - -# Instruction latencies and initiation intervals -# "ADD,MAX,MUL,MAD,DIV" --ptx_opcode_latency_int 4,13,4,5,145 --ptx_opcode_initiation_int 1,2,2,1,8 --ptx_opcode_latency_fp 4,13,4,5,39 --ptx_opcode_initiation_fp 1,2,1,1,4 --ptx_opcode_latency_dp 8,19,8,8,330 --ptx_opcode_initiation_dp 8,16,8,8,130 - - -# In Fermi, the cache and shared memory can be configured to 16kb:48kb(default) or 48kb:16kb -# ::,::::,::,:** -# ** Optional parameter - Required when mshr_type==Texture Fifo -# Note: Hashing set index function (H) only applies to a set size of 32 or 64. --gpgpu_cache:dl1 32:128:4,L:L:m:N:H,A:32:8,8 --gpgpu_shmem_size 49152 - -# The alternative configuration for fermi in case cudaFuncCachePreferL1 is selected -#-gpgpu_cache:dl1 64:128:6,L:L:m:N:H,A:32:8,8 -#-gpgpu_shmem_size 16384 - -# 64 sets, each 128 bytes 8-way for each memory sub partition. This gives 786KB L2 cache --gpgpu_cache:dl2 64:128:8,L:B:m:W:L,A:32:4,4:0,32 --gpgpu_cache:dl2_texture_only 0 - --gpgpu_cache:il1 4:128:4,L:R:f:N:L,A:2:32,4 --gpgpu_tex_cache:l1 4:128:24,L:R:m:N:L,F:128:4,128:2 --gpgpu_const_cache:l1 64:64:2,L:R:f:N:L,A:2:32,4 - -# enable operand collector --gpgpu_operand_collector_num_units_sp 6 --gpgpu_operand_collector_num_units_sfu 8 --gpgpu_operand_collector_num_in_ports_sp 2 --gpgpu_operand_collector_num_out_ports_sp 2 --gpgpu_num_reg_banks 16 - -# shared memory bankconflict detection --gpgpu_shmem_num_banks 32 --gpgpu_shmem_limited_broadcast 0 --gpgpu_shmem_warp_parts 1 - --gpgpu_max_insn_issue_per_warp 1 - -# interconnection --network_mode 1 --inter_config_file config_fermi_islip.icnt - -# memory partition latency config --rop_latency 120 --dram_latency 100 - -# dram model config --gpgpu_dram_scheduler 1 -# The DRAM return queue and the scheduler queue together should provide buffer -# to sustain the memory level parallelism to tolerate DRAM latency -# To allow 100% DRAM utility, there should at least be enough buffer to sustain -# the minimum DRAM latency (100 core cycles). I.e. -# Total buffer space required = 100 x 924MHz / 700MHz = 132 --gpgpu_frfcfs_dram_sched_queue_size 16 --gpgpu_dram_return_queue_size 116 - -# for Fermi, bus width is 384bits, this is 8 bytes (4 bytes at each DRAM chip) per memory partition --gpgpu_n_mem_per_ctrlr 2 --gpgpu_dram_buswidth 4 --gpgpu_dram_burst_length 8 --dram_data_command_freq_ratio 4 # GDDR5 is QDR --gpgpu_mem_address_mask 1 --gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.BBBCCCCB.CCSSSSSS - -# GDDR5 timing from hynix H5GQ1H24AFR -# to disable bank groups, set nbkgrp to 1 and tCCDL and tRTPL to 0 --gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=6:RCD=12:RAS=28:RP=12:RC=40: - CL=12:WL=4:CDLR=5:WR=12:nbkgrp=4:CCDL=3:RTPL=2" - -# Fermi has two schedulers per core --gpgpu_num_sched_per_core 2 -# Two Level Scheduler with active and pending pools -#-gpgpu_scheduler two_level_active:6:0:1 -# Loose round robbin scheduler -#-gpgpu_scheduler lrr -# Greedy then oldest scheduler --gpgpu_scheduler gto - -# stat collection --gpgpu_memlatency_stat 14 --gpgpu_runtime_stat 500 --enable_ptx_file_line_stats 1 --visualizer_enabled 0 - -# power model configs --power_simulation_enabled 1 --gpuwattch_xml_file gpuwattch_gtx480.xml - -# tracing functionality -#-trace_enabled 1 -#-trace_components WARP_SCHEDULER,SCOREBOARD -#-trace_sampling_core 0 diff --git a/configs/GTX480/gpuwattch_gtx480.xml b/configs/GTX480/gpuwattch_gtx480.xml deleted file mode 100755 index 304e0fd..0000000 --- a/configs/GTX480/gpuwattch_gtx480.xml +++ /dev/null @@ -1,538 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/configs/GeForceGTX1080Ti/config_fermi_islip.icnt b/configs/GeForceGTX1080Ti/config_fermi_islip.icnt deleted file mode 100644 index 2a69ddd..0000000 --- a/configs/GeForceGTX1080Ti/config_fermi_islip.icnt +++ /dev/null @@ -1,70 +0,0 @@ -//21*1 fly with 32 flits per packet under gpgpusim injection mode -use_map = 0; -flit_size = 32; - -// currently we do not use this, see subnets below -network_count = 2; - -// Topology -topology = fly; -k = 50; -n = 1; - -// Routing - -routing_function = dest_tag; - -// Flow control - -num_vcs = 1; -vc_buf_size = 8; - -wait_for_tail_credit = 0; - -// Router architecture - -vc_allocator = islip; //separable_input_first; -sw_allocator = islip; //separable_input_first; -alloc_iters = 1; - -credit_delay = 0; -routing_delay = 0; -vc_alloc_delay = 1; -sw_alloc_delay = 1; - -input_speedup = 2; -output_speedup = 1; -internal_speedup = 1.0; - -// Traffic, GPGPU-Sim does not use this - -traffic = uniform; -packet_size ={{1,2,3,4},{10,20}}; -packet_size_rate={{1,1,1,1},{2,1}}; - -// Simulation - Don't change - -sim_type = gpgpusim; -//sim_type = latency; -injection_rate = 0.1; - -subnets = 2; - -// Always use read and write no matter following line -//use_read_write = 1; - - -read_request_subnet = 0; -read_reply_subnet = 1; -write_request_subnet = 0; -write_reply_subnet = 1; - -read_request_begin_vc = 0; -read_request_end_vc = 0; -write_request_begin_vc = 0; -write_request_end_vc = 0; -read_reply_begin_vc = 0; -read_reply_end_vc = 0; -write_reply_begin_vc = 0; -write_reply_end_vc = 0; - diff --git a/configs/GeForceGTX1080Ti/gpgpusim.config b/configs/GeForceGTX1080Ti/gpgpusim.config deleted file mode 100644 index fb044c6..0000000 --- a/configs/GeForceGTX1080Ti/gpgpusim.config +++ /dev/null @@ -1,150 +0,0 @@ -# This config models the Pascal GP102 (GeForceGTX 1080Ti) - -# functional simulator specification --gpgpu_ptx_instruction_classification 0 --gpgpu_ptx_sim_mode 0 --gpgpu_ptx_force_max_capability 60 - -# SASS execution (only supported with CUDA >= 4.0) --gpgpu_ptx_convert_to_ptxplus 0 --gpgpu_ptx_save_converted_ptxplus 0 - -# high level architecture configuration --gpgpu_n_clusters 28 --gpgpu_n_cores_per_cluster 1 --gpgpu_n_mem 11 --gpgpu_n_sub_partition_per_mchannel 2 - -# Pascal clock domains -#-gpgpu_clock_domains ::: -# Pascal NVIDIA TITAN X clock domains are adopted from -# https://en.wikipedia.org/wiki/GeForce_10_series --gpgpu_clock_domains 1481.0:2962.0:1481.0:2750.0 - -# shader core pipeline config --gpgpu_shader_registers 65536 --gpgpu_occupancy_sm_number 60 - -# This implies a maximum of 64 warps/SM --gpgpu_shader_core_pipeline 2048:32 --gpgpu_shader_cta 32 --gpgpu_simd_model 1 - -# Pipeline widths and number of FUs -# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB -## Pascal GP102 has 4 SP SIMD units and 1 SFU unit -## we need to scale the number of pipeline registers to be equal to the number of SP units --gpgpu_pipeline_widths 4,1,1,4,1,1,6 --gpgpu_num_sp_units 4 --gpgpu_num_sfu_units 1 - -# Instruction latencies and initiation intervals -# "ADD,MAX,MUL,MAD,DIV" -# SFU is 32-width in pascal, then dp units initiation is 1 cycle --ptx_opcode_latency_int 4,13,4,5,145 --ptx_opcode_initiation_int 1,2,2,2,8 --ptx_opcode_latency_fp 4,13,4,5,39 --ptx_opcode_initiation_fp 1,2,1,1,4 --ptx_opcode_latency_dp 8,19,8,8,330 --ptx_opcode_initiation_dp 1,2,1,1,130 - -# ::,::::,::,:** -# ** Optional parameter - Required when mshr_type==Texture Fifo -# Note: Hashing set index function (H) only applies to a set size of 32 or 64. -# Pascal GP102 has 96KB Shared memory -# Pascal GP102 has 64KB L1 cache -# The default is to disable the L1 cache, unless cache modifieres is used --gpgpu_cache:dl1 64:128:6,L:L:m:N:H,A:128:8,8 --gpgpu_shmem_size 98304 --gmem_skip_L1D 1 - -# 64 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 3MB L2 cache --gpgpu_cache:dl2 64:128:16,L:B:m:W:L,A:1024:1024,4:0,32 # used to be 128:4 --gpgpu_cache:dl2_texture_only 0 - -# 4 KB Inst. --gpgpu_cache:il1 8:128:4,L:R:f:N:L,A:2:48,4 -# 48 KB Tex --gpgpu_tex_cache:l1 16:128:24,L:R:m:N:L,F:128:4,128:2 -# 12 KB Const --gpgpu_const_cache:l1 128:64:2,L:R:f:N:L,A:2:64,4 - -# enable operand collector -## larger operand collectors and reg_banks are needed for the 4 warp schedulers and 4 SIMD units --gpgpu_operand_collector_num_units_sp 20 --gpgpu_operand_collector_num_units_sfu 4 --gpgpu_operand_collector_num_units_mem 8 --gpgpu_operand_collector_num_in_ports_sp 4 --gpgpu_operand_collector_num_out_ports_sp 4 --gpgpu_operand_collector_num_in_ports_sfu 1 --gpgpu_operand_collector_num_out_ports_sfu 1 --gpgpu_operand_collector_num_in_ports_mem 1 --gpgpu_operand_collector_num_out_ports_mem 1 -# gpgpu_num_reg_banks should be increased to 32, but it gives an error! --gpgpu_num_reg_banks 32 - -# shared memory bankconflict detection --gpgpu_shmem_num_banks 32 --gpgpu_shmem_limited_broadcast 0 --gpgpu_shmem_warp_parts 1 - -## In Pascal, a warp scheduler can issue 2 insts per cycle --gpgpu_max_insn_issue_per_warp 2 - -# interconnection --network_mode 1 --inter_config_file config_fermi_islip.icnt - -# memory partition latency config --rop_latency 120 --dram_latency 100 - -# dram model config --gpgpu_dram_scheduler 1 -# The DRAM return queue and the scheduler queue together should provide buffer -# to sustain the memory level parallelism to tolerate DRAM latency -# To allow 100% DRAM utility, there should at least be enough buffer to sustain -# the minimum DRAM latency (100 core cycles). I.e. -# Total buffer space required = 100 x 924MHz / 700MHz = 132 --gpgpu_frfcfs_dram_sched_queue_size 64 --gpgpu_dram_return_queue_size 116 - -# for NVIDIA GeForceGTX 1080Ti, bus width is 352bits (11 DRAM chips x 32 bits) -# 11 memory paritions, 4 bytes (1 DRAM chip) per memory partition -# the atom size of GDDR5X (the smallest read request) is 32 bytes --gpgpu_n_mem_per_ctrlr 1 --gpgpu_dram_buswidth 4 --gpgpu_dram_burst_length 8 --dram_data_command_freq_ratio 4 # GDDR5X is QDR --gpgpu_mem_address_mask 1 --gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS - -# Use the same GDDR5 timing from hynix H5GQ1H24AFR -# disable bank groups for now, set nbkgrp to 1 and tCCDL and tRTPL to 0 --gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=6:RCD=12:RAS=28:RP=12:RC=40: - CL=12:WL=4:CDLR=5:WR=12:nbkgrp=1:CCDL=0:RTPL=0" - -# Pascal has four schedulers per core --gpgpu_num_sched_per_core 2 -# Two Level Scheduler with active and pending pools -#-gpgpu_scheduler two_level_active:6:0:1 -# Loose round robbin scheduler -#-gpgpu_scheduler lrr -# Greedy then oldest scheduler --gpgpu_scheduler gto - -# stat collection --gpgpu_memlatency_stat 14 --gpgpu_runtime_stat 500 --enable_ptx_file_line_stats 1 --visualizer_enabled 0 - -# power model configs --power_simulation_enabled 1 --gpuwattch_xml_file gpuwattch_gtx1080Ti.xml - -# tracing functionality -#-trace_enabled 1 -#-trace_components WARP_SCHEDULER,SCOREBOARD -#-trace_sampling_core 0 - diff --git a/configs/GeForceGTX1080Ti/gpuwattch_gtx1080Ti.xml b/configs/GeForceGTX1080Ti/gpuwattch_gtx1080Ti.xml deleted file mode 100755 index 02619ff..0000000 --- a/configs/GeForceGTX1080Ti/gpuwattch_gtx1080Ti.xml +++ /dev/null @@ -1,538 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/configs/GeForceGTX750Ti/config_fermi_islip.icnt b/configs/GeForceGTX750Ti/config_fermi_islip.icnt deleted file mode 100644 index 069ca02..0000000 --- a/configs/GeForceGTX750Ti/config_fermi_islip.icnt +++ /dev/null @@ -1,70 +0,0 @@ -//21*1 fly with 32 flits per packet under gpgpusim injection mode -use_map = 0; -flit_size = 32; - -// currently we do not use this, see subnets below -network_count = 2; - -// Topology -topology = fly; -k = 7; -n = 1; - -// Routing - -routing_function = dest_tag; - -// Flow control - -num_vcs = 1; -vc_buf_size = 8; - -wait_for_tail_credit = 0; - -// Router architecture - -vc_allocator = islip; //separable_input_first; -sw_allocator = islip; //separable_input_first; -alloc_iters = 1; - -credit_delay = 0; -routing_delay = 0; -vc_alloc_delay = 1; -sw_alloc_delay = 1; - -input_speedup = 2; -output_speedup = 1; -internal_speedup = 1.0; - -// Traffic, GPGPU-Sim does not use this - -traffic = uniform; -packet_size ={{1,2,3,4},{10,20}}; -packet_size_rate={{1,1,1,1},{2,1}}; - -// Simulation - Don't change - -sim_type = gpgpusim; -//sim_type = latency; -injection_rate = 0.1; - -subnets = 2; - -// Always use read and write no matter following line -//use_read_write = 1; - - -read_request_subnet = 0; -read_reply_subnet = 1; -write_request_subnet = 0; -write_reply_subnet = 1; - -read_request_begin_vc = 0; -read_request_end_vc = 0; -write_request_begin_vc = 0; -write_request_end_vc = 0; -read_reply_begin_vc = 0; -read_reply_end_vc = 0; -write_reply_begin_vc = 0; -write_reply_end_vc = 0; - diff --git a/configs/GeForceGTX750Ti/gpgpusim.config b/configs/GeForceGTX750Ti/gpgpusim.config deleted file mode 100644 index c675aab..0000000 --- a/configs/GeForceGTX750Ti/gpgpusim.config +++ /dev/null @@ -1,131 +0,0 @@ -# functional simulator specification --gpgpu_ptx_instruction_classification 0 --gpgpu_ptx_sim_mode 0 --gpgpu_ptx_force_max_capability 52 - - -# SASS execution (only supported with CUDA >= 4.0) --gpgpu_ptx_convert_to_ptxplus 0 --gpgpu_ptx_save_converted_ptxplus 0 - -# high level architecture configuration --gpgpu_n_clusters 5 --gpgpu_n_cores_per_cluster 1 --gpgpu_n_mem 2 --gpgpu_n_sub_partition_per_mchannel 1 - -# Maxwell clock domains -#-gpgpu_clock_domains ::: -# In Maxwell, each pipeline has 32 execution units, so the Core clock is just the reported number. --gpgpu_clock_domains 1137.0:1137.0:1137.0:2700.0 - -# shader core pipeline config --gpgpu_shader_registers 65536 --gpgpu_occupancy_sm_number 52 - -# This implies a maximum of 64 warps/SM --gpgpu_shader_core_pipeline 2048:32 --gpgpu_shader_cta 8 --gpgpu_simd_model 1 - -# Pipeline widths and number of FUs -# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB --gpgpu_pipeline_widths 2,1,1,2,1,1,2 --gpgpu_num_sp_units 8 --gpgpu_num_sfu_units 32 - -# Instruction latencies and initiation intervals -# "ADD,MAX,MUL,MAD,DIV" --ptx_opcode_latency_int 6,12,13,13,210 --ptx_opcode_initiation_int 1,1,1,1,4 --ptx_opcode_latency_fp 6,12,6,6,374 --ptx_opcode_initiation_fp 1,1,1,1,4 --ptx_opcode_latency_dp 8,19,8,8,330 --ptx_opcode_initiation_dp 8,16,8,8,130 - --gpgpu_cache:dl1 32:128:4,L:L:m:N:H,A:32:8,8 --gmem_skip_L1D 1 --gpgpu_shmem_size 65536 - -# The alternative configuration for fermi in case cudaFuncCachePreferL1 is selected -#-gpgpu_cache:dl1 64:128:6,L:L:m:N:H,A:32:8,8 -#-gpgpu_shmem_size 16384 - -# 1024 sets, each 128 bytes 8-way for each memory sub partition. This gives 2MB L2 cache --gpgpu_cache:dl2 1024:128:8,L:B:m:W:L,A:32:4,4:0,32 --gpgpu_cache:dl2_texture_only 0 - --gpgpu_cache:il1 4:128:4,L:R:f:N:L,A:2:32,4 --gpgpu_tex_cache:l1 16:128:32,L:R:m:N:L,F:128:4,128:2 --gpgpu_const_cache:l1 64:64:2,L:R:f:N:L,A:2:32,4 - -# enable operand collector --gpgpu_operand_collector_num_units_sp 6 --gpgpu_operand_collector_num_units_sfu 8 --gpgpu_operand_collector_num_in_ports_sp 2 --gpgpu_operand_collector_num_out_ports_sp 2 --gpgpu_num_reg_banks 16 - -# shared memory bankconflict detection --gpgpu_shmem_num_banks 32 --gpgpu_shmem_limited_broadcast 0 --gpgpu_shmem_warp_parts 1 - --gpgpu_max_insn_issue_per_warp 2 - -# interconnection --network_mode 1 --inter_config_file config_fermi_islip.icnt - -# memory partition latency config --rop_latency 150 --dram_latency 130 - -# dram model config --gpgpu_dram_scheduler 1 -# The DRAM return queue and the scheduler queue together should provide buffer -# to sustain the memory level parallelism to tolerate DRAM latency -# To allow 100% DRAM utility, there should at least be enough buffer to sustain -# the minimum DRAM latency (100 core cycles). I.e. -# Total buffer space required = 100 x 924MHz / 700MHz = 132 --gpgpu_frfcfs_dram_sched_queue_size 16 --gpgpu_dram_return_queue_size 300 - -# for Maxwell, bus width is 128bits, this is 64 bytes (32 bytes at each DRAM chip) per memory partition --gpgpu_n_mem_per_ctrlr 2 --gpgpu_dram_buswidth 32 --gpgpu_dram_burst_length 8 --dram_data_command_freq_ratio 4 # GDDR5 is QDR --gpgpu_mem_address_mask 1 --gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.BBBCCCCB.CCSSSSSS - -# GDDR5 timing from hynix H5GQ1H24AFR -# to disable bank groups, set nbkgrp to 1 and tCCDL and tRTPL to 0 --gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=6:RCD=12:RAS=28:RP=12:RC=40: - CL=12:WL=4:CDLR=5:WR=12:nbkgrp=4:CCDL=3:RTPL=2" - -# Maxwell has four schedulers per core --gpgpu_num_sched_per_core 4 -# Two Level Scheduler with active and pending pools -#-gpgpu_scheduler two_level_active:6:0:1 -# Loose round robbin scheduler -#-gpgpu_scheduler lrr -# Greedy then oldest scheduler --gpgpu_scheduler gto - -# stat collection --gpgpu_memlatency_stat 14 --gpgpu_runtime_stat 500 --enable_ptx_file_line_stats 1 --visualizer_enabled 0 - -# power model configs -# power modeling is currently disabled for GTX750Ti. The gpuwattch_gtx750ti.xml file present is only a placeholder and has NOT been tested for accuracy. -# proper modeling of Maxwell power is planned, but should be considered completely unsupported at present. --power_simulation_enabled 0 --gpuwattch_xml_file gpuwattch_gtx750Ti.xml - -# tracing functionality -#-trace_enabled 1 -#-trace_components WARP_SCHEDULER,SCOREBOARD -#-trace_sampling_core 0 diff --git a/configs/GeForceGTX750Ti/gpuwattch_gtx750Ti.xml b/configs/GeForceGTX750Ti/gpuwattch_gtx750Ti.xml deleted file mode 100755 index e2b2324..0000000 --- a/configs/GeForceGTX750Ti/gpuwattch_gtx750Ti.xml +++ /dev/null @@ -1,539 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/configs/Pascal_TITANX/config_fermi_islip.icnt b/configs/Pascal_TITANX/config_fermi_islip.icnt deleted file mode 100644 index 602daee..0000000 --- a/configs/Pascal_TITANX/config_fermi_islip.icnt +++ /dev/null @@ -1,70 +0,0 @@ -//21*1 fly with 32 flits per packet under gpgpusim injection mode -use_map = 0; -flit_size = 32; - -// currently we do not use this, see subnets below -network_count = 2; - -// Topology -topology = fly; -k = 52; -n = 1; - -// Routing - -routing_function = dest_tag; - -// Flow control - -num_vcs = 1; -vc_buf_size = 8; - -wait_for_tail_credit = 0; - -// Router architecture - -vc_allocator = islip; //separable_input_first; -sw_allocator = islip; //separable_input_first; -alloc_iters = 1; - -credit_delay = 0; -routing_delay = 0; -vc_alloc_delay = 1; -sw_alloc_delay = 1; - -input_speedup = 2; -output_speedup = 1; -internal_speedup = 1.0; - -// Traffic, GPGPU-Sim does not use this - -traffic = uniform; -packet_size ={{1,2,3,4},{10,20}}; -packet_size_rate={{1,1,1,1},{2,1}}; - -// Simulation - Don't change - -sim_type = gpgpusim; -//sim_type = latency; -injection_rate = 0.1; - -subnets = 2; - -// Always use read and write no matter following line -//use_read_write = 1; - - -read_request_subnet = 0; -read_reply_subnet = 1; -write_request_subnet = 0; -write_reply_subnet = 1; - -read_request_begin_vc = 0; -read_request_end_vc = 0; -write_request_begin_vc = 0; -write_request_end_vc = 0; -read_reply_begin_vc = 0; -read_reply_end_vc = 0; -write_reply_begin_vc = 0; -write_reply_end_vc = 0; - diff --git a/configs/Pascal_TITANX/gpgpusim.config b/configs/Pascal_TITANX/gpgpusim.config deleted file mode 100644 index f78bd02..0000000 --- a/configs/Pascal_TITANX/gpgpusim.config +++ /dev/null @@ -1,156 +0,0 @@ -# This config models the Pascal GP102 (NVIDIA TITAN X) -# For more info about this card, see Nvidia White paper -# http://international.download.nvidia.com/geforce-com/international/pdfs/GeForce_GTX_1080_Whitepaper_FINAL.pdf - -# functional simulator specification --gpgpu_ptx_instruction_classification 0 --gpgpu_ptx_sim_mode 0 --gpgpu_ptx_force_max_capability 61 - -# SASS execution (only supported with CUDA >= 4.0) --gpgpu_ptx_convert_to_ptxplus 0 --gpgpu_ptx_save_converted_ptxplus 0 - -# high level architecture configuration --gpgpu_n_clusters 28 --gpgpu_n_cores_per_cluster 1 --gpgpu_n_mem 12 --gpgpu_n_sub_partition_per_mchannel 2 - -# Pascal clock domains -#-gpgpu_clock_domains ::: -# Pascal NVIDIA TITAN X clock domains are adopted from -# https://en.wikipedia.org/wiki/GeForce_10_series --gpgpu_clock_domains 1417.0:1417.0:1417.0:2500.0 - -# shader core pipeline config --gpgpu_shader_registers 65536 - -# This implies a maximum of 64 warps/SM --gpgpu_shader_core_pipeline 2048:32 --gpgpu_shader_cta 32 --gpgpu_simd_model 1 - -# Pipeline widths and number of FUs -# ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB -## Pascal GP102 has 4 SP SIMD units and 1 SFU unit -## we need to scale the number of pipeline registers to be equal to the number of SP units --gpgpu_pipeline_widths 4,1,1,4,1,1,6 --gpgpu_num_sp_units 4 --gpgpu_num_sfu_units 1 - -# Instruction latencies and initiation intervals -# "ADD,MAX,MUL,MAD,DIV" -# SFU is 32-width in pascal, then dp units initiation is 1 cycle --ptx_opcode_latency_int 4,13,4,5,145 --ptx_opcode_initiation_int 1,1,1,1,4 --ptx_opcode_latency_fp 4,13,4,5,39 --ptx_opcode_initiation_fp 1,2,1,1,4 --ptx_opcode_latency_dp 8,19,8,8,330 --ptx_opcode_initiation_dp 4,8,4,4,130 - -# ::,::::,::,:** -# ** Optional parameter - Required when mshr_type==Texture Fifo -# Note: Hashing set index function (H) only applies to a set size of 32 or 64. -# Pascal GP102 has 96KB Shared memory -# Pascal GP102 has 64KB L1 cache -# The defulat is to disable the L1 cache, unless cache modifieres is used --gpgpu_cache:dl1 64:128:6,L:L:m:N:H,A:128:8,16 --gpgpu_cache:dl1PrefL1 64:128:6,L:L:m:N:H,A:128:8,16 --gpgpu_cache:dl1PrefShared 64:128:6,L:L:m:N:H,A:128:8,16 --gpgpu_shmem_size 98304 --gpgpu_shmem_size_PrefL1 98304 --gpgpu_shmem_size_PrefShared 98304 --gmem_skip_L1D 1 - -# 64 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 3MB L2 cache --gpgpu_cache:dl2 64:128:16,L:B:m:W:L,A:128:8,4:0,32 --gpgpu_cache:dl2_texture_only 0 --gpgpu_dram_partition_queues 32:32:32:32 - -# 4 KB Inst. --gpgpu_cache:il1 8:128:4,L:R:f:N:L,A:2:48,4 -# 48 KB Tex --gpgpu_tex_cache:l1 16:128:24,L:R:m:N:L,F:128:4,128:2 -# 12 KB Const --gpgpu_const_cache:l1 128:64:2,L:R:f:N:L,A:2:64,4 - -# enable operand collector -## larger operand collectors and reg_banks are needed for the 4 warp schedulers and 4 SIMD units --gpgpu_operand_collector_num_units_sp 20 --gpgpu_operand_collector_num_units_sfu 4 --gpgpu_operand_collector_num_units_mem 8 --gpgpu_operand_collector_num_in_ports_sp 4 --gpgpu_operand_collector_num_out_ports_sp 4 --gpgpu_operand_collector_num_in_ports_sfu 1 --gpgpu_operand_collector_num_out_ports_sfu 1 --gpgpu_operand_collector_num_in_ports_mem 1 --gpgpu_operand_collector_num_out_ports_mem 1 -# gpgpu_num_reg_banks should be increased to 32 --gpgpu_num_reg_banks 32 - -# shared memory bankconflict detection --gpgpu_shmem_num_banks 32 --gpgpu_shmem_limited_broadcast 0 --gpgpu_shmem_warp_parts 1 - -## In Pascal, a warp scheduler can issue 2 insts per cycle --gpgpu_max_insn_issue_per_warp 2 - -# interconnection --network_mode 1 --inter_config_file config_fermi_islip.icnt - -# memory partition latency config --rop_latency 120 --dram_latency 100 - -# dram model config --gpgpu_dram_scheduler 1 -# The DRAM return queue and the scheduler queue together should provide buffer -# to sustain the memory level parallelism to tolerate DRAM latency -# To allow 100% DRAM utility, there should at least be enough buffer to sustain -# the minimum DRAM latency (100 core cycles). I.e. -# Total buffer space required = 100 x 924MHz / 700MHz = 132 --gpgpu_frfcfs_dram_sched_queue_size 64 --gpgpu_dram_return_queue_size 116 - -# for NVIDIA TITAN X, bus width is 384bits (12 DRAM chips x 32 bits) -# 12 memory paritions, 4 bytes (1 DRAM chip) per memory partition -# the atom size of GDDR5X (the smallest read request) is 32 bytes --gpgpu_n_mem_per_ctrlr 1 --gpgpu_dram_buswidth 4 --gpgpu_dram_burst_length 8 --dram_data_command_freq_ratio 4 # GDDR5X is QDR --gpgpu_mem_address_mask 1 --gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS - -# Use the same GDDR5 timing from hynix H5GQ1H24AFR -# disable bank groups for now, set nbkgrp to 1 and tCCDL and tRTPL to 0 --gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=6:RCD=12:RAS=28:RP=12:RC=40: - CL=12:WL=4:CDLR=5:WR=12:nbkgrp=1:CCDL=0:RTPL=0" - -# Fermi has two schedulers per core --gpgpu_num_sched_per_core 4 -# Two Level Scheduler with active and pending pools -#-gpgpu_scheduler two_level_active:6:0:1 -# Loose round robbin scheduler -#-gpgpu_scheduler lrr -# Greedy then oldest scheduler --gpgpu_scheduler gto - -# stat collection --gpgpu_memlatency_stat 14 --gpgpu_runtime_stat 500 --enable_ptx_file_line_stats 1 --visualizer_enabled 0 - -# power model configs --power_simulation_enabled 1 --gpuwattch_xml_file gpuwattch_gtx480.xml - -# tracing functionality -#-trace_enabled 1 -#-trace_components WARP_SCHEDULER,SCOREBOARD -#-trace_sampling_core 0 - diff --git a/configs/QuadroFX5600/gpgpusim.config b/configs/QuadroFX5600/gpgpusim.config deleted file mode 100644 index 6f836ee..0000000 --- a/configs/QuadroFX5600/gpgpusim.config +++ /dev/null @@ -1,99 +0,0 @@ -# functional simulator specification --gpgpu_ptx_instruction_classification 0 --gpgpu_ptx_sim_mode 0 --gpgpu_ptx_force_max_capability 12 - -# high level architecture configuration --gpgpu_n_clusters 8 --gpgpu_n_cores_per_cluster 2 --gpgpu_n_mem 6 --gpgpu_clock_domains 337.5:600.0:600.0:800.0 - -# shader core pipeline config --gpgpu_shader_registers 16384 --gpgpu_occupancy_sm_number 12 - --gpgpu_occupancy_sm_number 12 -#8192 (registers per block as written by device Query and which used in this option in our other configurations but this break some benchmarks execution! it does not affect performance modeling though) --gpgpu_shader_core_pipeline 768:32 --gpgpu_shader_cta 8 --gpgpu_simd_model 1 - -# Pipeline widths and number of FUs -# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB --gpgpu_pipeline_widths 1,1,1,1,1,1,1 --gpgpu_num_sp_units 1 --gpgpu_num_sfu_units 1 - -# Instruction latencies and initiation intervals -# "ADD,MAX,MUL,MAD,DIV" --ptx_opcode_latency_int 1,1,19,25,145 --ptx_opcode_initiation_int 1,1,4,4,32 --ptx_opcode_latency_fp 1,1,1,1,30 --ptx_opcode_initiation_fp 1,1,1,1,5 --ptx_opcode_latency_dp 8,8,8,8,335 --ptx_opcode_initiation_dp 8,8,8,8,130 - -# memory stage behaviour --gpgpu_cache:il1 4:256:4,L:R:f:N:L,A:2:32,4 --gpgpu_tex_cache:l1 8:128:5,L:R:m:N:L,F:128:4,128:2 --gpgpu_const_cache:l1 64:64:2,L:R:f:N:L,A:2:32,4 --gpgpu_cache:dl2 16:256:8,L:B:m:W:L,A:16:4,4 --gpgpu_cache:dl2_texture_only 1 - -# TLB parameters -#-gpgpu_cache:tlbl1 1:524288:16:1:L:R:m,A:32:8,8 -#-gpgpu_tlbl2_latency 45 - - --gpgpu_shmem_warp_parts 2 - -# interconnection --network_mode 1 --inter_config_file icnt_config_islip.icnt - -# dram scheduler config --gpgpu_dram_scheduler 1 -# The DRAM return queue and the scheduler queue together should provide buffer -# to sustain the memory level parallelism to tolerate DRAM latency -# To allow 100% DRAM utility, there should at least be enough buffer to sustain -# the minimum DRAM latency (30 core cycles). I.e. -# Total buffer space required = 30 x 800MHz / 337.5MHz = 71 --gpgpu_frfcfs_dram_sched_queue_size 16 --gpgpu_dram_return_queue_size 55 - -# dram model config --gpgpu_n_mem_per_ctrlr 2 --gpgpu_dram_buswidth 4 --gpgpu_dram_burst_length 4 --gpgpu_mem_address_mask 1 --gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RRBBBCCC.CCCSSSSS -# GDDR3 timing from Samsung K4J52324QH-HC12 @ 800MHz -# {nbk:tCCD:tRRD:tRCD:tRAS:tRP:tRC:CL:WL:tCDLR:tWR:nbkgrp:tCCDL:tRTPL} --gpgpu_dram_timing_opt nbk=8:CCD=2:RRD=8:RCD=12:RAS=25:RP=10:RC=35:CL=10:WL=7:CDLR=6:WR=11 - -# stat collection --gpgpu_memlatency_stat 14 --gpgpu_runtime_stat 500 --enable_ptx_file_line_stats 1 - - - -# Using cuobjdump to extract ptx/SASS --gpgpu_ptx_use_cuobjdump 1 - -# SASS execution (only supported with CUDA >= 4.0) --gpgpu_ptx_convert_to_ptxplus 0 --gpgpu_ptx_save_converted_ptxplus 0 - -# enable operand collector --gpgpu_operand_collector_num_units_sp 6 --gpgpu_operand_collector_num_units_sfu 8 - --visualizer_enabled 0 --power_trace_enabled 0 --power_simulation_enabled 1 --gpuwattch_xml_file gpuwattch_quadrofx5600.xml - --steady_power_levels_enabled 1 --steady_state_definition 8,4 diff --git a/configs/QuadroFX5600/gpuwattch_quadrofx5600.xml b/configs/QuadroFX5600/gpuwattch_quadrofx5600.xml deleted file mode 100644 index 2c5a6fc..0000000 --- a/configs/QuadroFX5600/gpuwattch_quadrofx5600.xml +++ /dev/null @@ -1,538 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/configs/QuadroFX5600/icnt_config_islip.icnt b/configs/QuadroFX5600/icnt_config_islip.icnt deleted file mode 100644 index de3bcc8..0000000 --- a/configs/QuadroFX5600/icnt_config_islip.icnt +++ /dev/null @@ -1,70 +0,0 @@ -//14*1 fly with 32 flits per packet under gpgpusim injection mode -use_map = 0; -flit_size = 32; - -// currently we donot use this, see subnets below -network_count = 2; - -// Topology -topology = fly; -k = 14; -n = 1; - -// Routing - -routing_function = dest_tag; - -// Flow control - -num_vcs = 1; -vc_buf_size = 8; - -wait_for_tail_credit = 0; - -// Router architecture - -vc_allocator = islip; //separable_input_first; -sw_allocator = islip; //separable_input_first; -alloc_iters = 1; - -credit_delay = 0; -routing_delay = 0; -vc_alloc_delay = 1; -sw_alloc_delay = 1; - -input_speedup = 2; -output_speedup = 1; -internal_speedup = 1.0; - -// Traffic, GPGPU-Sim does not use this - -traffic = uniform; -packet_size ={{1,2,3,4},{10,20}}; -packet_size_rate={{1,1,1,1},{2,1}}; - -// Simulation - Don't change - -sim_type = gpgpusim; -//sim_type = latency; -injection_rate = 0.1; - -subnets = 2; - -// Always use read and write no matter following line -//use_read_write = 1; - - -read_request_subnet = 0; -read_reply_subnet = 1; -write_request_subnet = 0; -write_reply_subnet = 1; - -read_request_begin_vc = 0; -read_request_end_vc = 0; -write_request_begin_vc = 0; -write_request_end_vc = 0; -read_reply_begin_vc = 0; -read_reply_end_vc = 0; -write_reply_begin_vc = 0; -write_reply_end_vc = 0; - diff --git a/configs/QuadroFX5800/config_quadro_islip.icnt b/configs/QuadroFX5800/config_quadro_islip.icnt deleted file mode 100644 index cfe9cac..0000000 --- a/configs/QuadroFX5800/config_quadro_islip.icnt +++ /dev/null @@ -1,69 +0,0 @@ -//18*1 fly with 32 flits per packet under gpgpusim injection mode -use_map = 0; -flit_size = 32; - -// currently we donot use this, see subnets below -network_count = 2; - -// Topology -topology = fly; -k = 18; -n = 1; - -// Routing - -routing_function = dest_tag; - -// Flow control - -num_vcs = 1; -vc_buf_size = 8; - -wait_for_tail_credit = 0; - -// Router architecture - -vc_allocator = islip; //separable_input_first; -sw_allocator = islip; //separable_input_first; -alloc_iters = 1; - -credit_delay = 0; -routing_delay = 0; -vc_alloc_delay = 1; -sw_alloc_delay = 1; - -input_speedup = 2; -output_speedup = 1; -internal_speedup = 1.0; - -// Traffic, GPGPU-Sim does not use this -traffic = uniform; -packet_size ={{1,2,3,4},{10,20}}; -packet_size_rate={{1,1,1,1},{2,1}}; - -// Simulation - Don't change -sim_type = gpgpusim; -//sim_type = latency; -injection_rate = 0.1; - - -subnets = 2; - -// Always use read and write no matter following line -//use_read_write = 1; - - -read_request_subnet = 0; -read_reply_subnet = 1; -write_request_subnet = 0; -write_reply_subnet = 1; - -read_request_begin_vc = 0; -read_request_end_vc = 0; -write_request_begin_vc = 0; -write_request_end_vc = 0; -read_reply_begin_vc = 0; -read_reply_end_vc = 0; -write_reply_begin_vc = 0; -write_reply_end_vc = 0; - diff --git a/configs/QuadroFX5800/gpgpusim.config b/configs/QuadroFX5800/gpgpusim.config deleted file mode 100644 index fef1110..0000000 --- a/configs/QuadroFX5800/gpgpusim.config +++ /dev/null @@ -1,88 +0,0 @@ -# functional simulator specification --gpgpu_ptx_instruction_classification 0 --gpgpu_ptx_sim_mode 0 --gpgpu_ptx_force_max_capability 13 - -# high level architecture configuration --gpgpu_n_clusters 10 --gpgpu_n_cores_per_cluster 3 --gpgpu_n_mem 8 --gpgpu_clock_domains 325.0:650.0:650.0:800.0 - -# shader core pipeline config --gpgpu_shader_registers 16384 --gpgpu_occupancy_sm_number 13 - --gpgpu_shader_core_pipeline 1024:32 --gpgpu_shader_cta 8 --gpgpu_simd_model 1 - -# Pipeline widths and number of FUs -# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB --gpgpu_pipeline_widths 1,1,1,1,1,1,1 --gpgpu_num_sp_units 1 --gpgpu_num_sfu_units 1 - -# Instruction latencies and initiation intervals -# "ADD,MAX,MUL,MAD,DIV" --ptx_opcode_latency_int 1,1,19,25,145 --ptx_opcode_initiation_int 1,1,4,4,32 --ptx_opcode_latency_fp 1,1,1,1,30 --ptx_opcode_initiation_fp 1,1,1,1,5 --ptx_opcode_latency_dp 8,8,8,8,335 --ptx_opcode_initiation_dp 8,8,8,8,130 - -# memory stage behaviour -# ::,::::,::,:** -# ** Optional parameter - Required when mshr_type==Texture Fifo --gpgpu_cache:il1 4:256:4,L:R:f:N:L,A:2:32,4 --gpgpu_tex_cache:l1 8:128:5,L:R:m:N:L,F:128:4,128:2 --gpgpu_const_cache:l1 64:64:2,L:R:f:N:L,A:2:32,4 --gpgpu_cache:dl2 16:256:8,L:B:m:W:L,A:16:4,4 --gpgpu_cache:dl2_texture_only 1 - --gpgpu_shmem_warp_parts 2 - -# interconnection --network_mode 1 --inter_config_file config_quadro_islip.icnt - -# dram scheduler config --gpgpu_dram_scheduler 1 -# The DRAM return queue and the scheduler queue together should provide buffer -# to sustain the memory level parallelism to tolerate DRAM latency -# To allow 100% DRAM utility, there should at least be enough buffer to sustain -# the minimum DRAM latency (30 core cycles). I.e. -# Total buffer space required = 30 x 800MHz / 325MHz = 74 --gpgpu_frfcfs_dram_sched_queue_size 16 --gpgpu_dram_return_queue_size 58 - -# dram model config --gpgpu_n_mem_per_ctrlr 2 --gpgpu_dram_buswidth 4 --gpgpu_dram_burst_length 4 --dram_data_command_freq_ratio 2 # GDDR3 is DDR --gpgpu_mem_address_mask 1 --gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RRBBBCCC.CCCSSSSS -# GDDR3 timing from Samsung K4J52324QH-HC12 @ 800MHz --gpgpu_dram_timing_opt nbk=8:CCD=2:RRD=8:RCD=12:RAS=25:RP=10:RC=35:CL=10:WL=7:CDLR=6:WR=11 - -# stat collection --gpgpu_memlatency_stat 14 --gpgpu_runtime_stat 500 --enable_ptx_file_line_stats 1 - -# SASS execution (only supported with CUDA >= 4.0) --gpgpu_ptx_convert_to_ptxplus 0 --gpgpu_ptx_save_converted_ptxplus 0 - -# enable operand collector --gpgpu_operand_collector_num_units_sp 6 --gpgpu_operand_collector_num_units_sfu 8 - --visualizer_enabled 0 - -# tracing functionality -#-trace_enabled 1 -#-trace_components WARP_SCHEDULER,SCOREBOARD -#-trace_sampling_core 0 diff --git a/configs/TeslaC2050/config_fermi_islip.icnt b/configs/TeslaC2050/config_fermi_islip.icnt deleted file mode 100644 index a11bd8e..0000000 --- a/configs/TeslaC2050/config_fermi_islip.icnt +++ /dev/null @@ -1,70 +0,0 @@ -//20*1 fly with 32 flits per packet under gpgpusim injection mode -use_map = 0; -flit_size = 32; - -// currently we donot use this, see subnets below -network_count = 2; - -// Topology -topology = fly; -k = 26; -n = 1; - -// Routing - -routing_function = dest_tag; - -// Flow control - -num_vcs = 1; -vc_buf_size = 8; - -wait_for_tail_credit = 0; - -// Router architecture - -vc_allocator = islip; //separable_input_first; -sw_allocator = islip; //separable_input_first; -alloc_iters = 1; - -credit_delay = 0; -routing_delay = 0; -vc_alloc_delay = 1; -sw_alloc_delay = 1; - -input_speedup = 2; -output_speedup = 1; -internal_speedup = 1.0; - -// Traffic, GPGPU-Sim does not use this - -traffic = uniform; -packet_size ={{1,2,3,4},{10,20}}; -packet_size_rate={{1,1,1,1},{2,1}}; - -// Simulation - Don't change - -sim_type = gpgpusim; -//sim_type = latency; -injection_rate = 0.1; - - -subnets = 2; - -// Always use read and write no matter following line -//use_read_write = 1; - - -read_request_subnet = 0; -read_reply_subnet = 1; -write_request_subnet = 0; -write_reply_subnet = 1; - -read_request_begin_vc = 0; -read_request_end_vc = 0; -write_request_begin_vc = 0; -write_request_end_vc = 0; -read_reply_begin_vc = 0; -read_reply_end_vc = 0; -write_reply_begin_vc = 0; -write_reply_end_vc = 0; diff --git a/configs/TeslaC2050/gpgpusim.config b/configs/TeslaC2050/gpgpusim.config deleted file mode 100644 index 6ac2c12..0000000 --- a/configs/TeslaC2050/gpgpusim.config +++ /dev/null @@ -1,133 +0,0 @@ -# functional simulator specification --gpgpu_ptx_instruction_classification 0 --gpgpu_ptx_sim_mode 0 --gpgpu_ptx_force_max_capability 20 - -# Using cuobjdump to extract ptx/SASS -#-gpgpu_ptx_use_cuobjdump 1 # use default - -# SASS execution (only supported with CUDA >= 4.0) --gpgpu_ptx_convert_to_ptxplus 0 --gpgpu_ptx_save_converted_ptxplus 0 - - -# high level architecture configuration --gpgpu_n_clusters 14 --gpgpu_n_cores_per_cluster 1 --gpgpu_n_mem 6 --gpgpu_n_sub_partition_per_mchannel 2 - -# Fermi clock domains -#-gpgpu_clock_domains ::: -# In Fermi, each pipeline has 16 execution units, so the Core clock needs to be divided -# by 2. (GPGPU-Sim simulates a warp (32 threads) in a single cycle). 1400/2 = 700 --gpgpu_clock_domains 575.0:575.0:575.0:750.0 - -# shader core pipeline config --gpgpu_shader_registers 32768 --gpgpu_occupancy_sm_number 20 - -# This implies a maximum of 48 warps/SM --gpgpu_shader_core_pipeline 1536:32 --gpgpu_shader_cta 8 --gpgpu_simd_model 1 - -# Pipeline widths and number of FUs -# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB --gpgpu_pipeline_widths 2,1,1,2,1,1,2 --gpgpu_num_sp_units 2 --gpgpu_num_sfu_units 1 - -# Instruction latencies and initiation intervals -# "ADD,MAX,MUL,MAD,DIV" --ptx_opcode_latency_int 4,13,4,5,145 --ptx_opcode_initiation_int 1,2,2,1,8 --ptx_opcode_latency_fp 4,13,4,5,39 --ptx_opcode_initiation_fp 1,2,1,1,4 --ptx_opcode_latency_dp 8,19,8,8,330 --ptx_opcode_initiation_dp 8,16,8,8,130 - - -# In Fermi, the cache and shared memory can be configured to 16kb:48kb(default) or 48kb:16kb -# ::,::::,::,:** -# ** Optional parameter - Required when mshr_type==Texture Fifo -# Note: Hashing set index function (H) only applies to a set size of 32 or 64. --gpgpu_cache:dl1 32:128:4,L:L:m:N:H,A:32:8,8 --gpgpu_shmem_size 49152 - -# The alternative configuration for fermi in case cudaFuncCachePreferL1 is selected -#-gpgpu_cache:dl1 64:128:6,L:L:m:N:H,A:32:8,8 -#-gpgpu_shmem_size 16384 - -# 64 sets, each 128 bytes 8-way for each memory sub partition. This gives 786KB L2 cache --gpgpu_cache:dl2 64:128:8,L:B:m:W:L,A:32:4,4:0,32 --gpgpu_cache:dl2_texture_only 0 - --gpgpu_cache:il1 4:128:4,L:R:f:N:L,A:2:32,4 --gpgpu_tex_cache:l1 4:128:24,L:R:m:N:L,F:128:4,128:2 --gpgpu_const_cache:l1 64:64:2,L:R:f:N:L,A:2:32,4 - -# enable operand collector --gpgpu_operand_collector_num_units_sp 6 --gpgpu_operand_collector_num_units_sfu 8 --gpgpu_operand_collector_num_in_ports_sp 2 --gpgpu_operand_collector_num_out_ports_sp 2 --gpgpu_num_reg_banks 16 - -# shared memory bankconflict detection --gpgpu_shmem_num_banks 32 --gpgpu_shmem_limited_broadcast 0 --gpgpu_shmem_warp_parts 1 - --gpgpu_max_insn_issue_per_warp 1 - -# interconnection --network_mode 1 --inter_config_file config_fermi_islip.icnt - -# memory partition latency config --rop_latency 120 --dram_latency 100 - -# dram model config --gpgpu_dram_scheduler 1 -# The DRAM return queue and the scheduler queue together should provide buffer -# to sustain the memory level parallelism to tolerate DRAM latency -# To allow 100% DRAM utility, there should at least be enough buffer to sustain -# the minimum DRAM latency (100 core cycles). I.e. -# Total buffer space required = 100 x 750MHz / 575MHz = 130 --gpgpu_frfcfs_dram_sched_queue_size 16 --gpgpu_dram_return_queue_size 114 - -# for Fermi, bus width is 384bits, this is 8 bytes (4 bytes at each DRAM chip) per memory partition --gpgpu_n_mem_per_ctrlr 2 --gpgpu_dram_buswidth 4 --gpgpu_dram_burst_length 8 --dram_data_command_freq_ratio 4 # GDDR5 is QDR --gpgpu_mem_address_mask 1 --gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.BBBCCCCB.CCSSSSSS - -# GDDR5 timing from hynix H5GQ1H24AFR -# to disable bank groups, set nbkgrp to 1 and tCCDL and tRTPL to 0 --gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=6:RCD=12:RAS=28:RP=12:RC=40: - CL=12:WL=4:CDLR=5:WR=12:nbkgrp=4:CCDL=3:RTPL=2" - -# Fermi has two schedulers per core --gpgpu_num_sched_per_core 2 -# Two Level Scheduler with active and pending pools -#-gpgpu_scheduler two_level_active:6:0:1 -# Loose round robbin scheduler -#-gpgpu_scheduler lrr -# Greedy then oldest scheduler --gpgpu_scheduler gto - -# stat collection --gpgpu_memlatency_stat 14 --gpgpu_runtime_stat 500 --enable_ptx_file_line_stats 1 --visualizer_enabled 0 - -# tracing functionality -#-trace_enabled 1 -#-trace_components WARP_SCHEDULER,SCOREBOARD -#-trace_sampling_core 0 diff --git a/configs/deprecated-cfgs/GTX480/config_fermi_islip.icnt b/configs/deprecated-cfgs/GTX480/config_fermi_islip.icnt new file mode 100644 index 0000000..7820e4e --- /dev/null +++ b/configs/deprecated-cfgs/GTX480/config_fermi_islip.icnt @@ -0,0 +1,70 @@ +//21*1 fly with 32 flits per packet under gpgpusim injection mode +use_map = 0; +flit_size = 32; + +// currently we do not use this, see subnets below +network_count = 2; + +// Topology +topology = fly; +k = 27; +n = 1; + +// Routing + +routing_function = dest_tag; + +// Flow control + +num_vcs = 1; +vc_buf_size = 8; + +wait_for_tail_credit = 0; + +// Router architecture + +vc_allocator = islip; //separable_input_first; +sw_allocator = islip; //separable_input_first; +alloc_iters = 1; + +credit_delay = 0; +routing_delay = 0; +vc_alloc_delay = 1; +sw_alloc_delay = 1; + +input_speedup = 2; +output_speedup = 1; +internal_speedup = 1.0; + +// Traffic, GPGPU-Sim does not use this + +traffic = uniform; +packet_size ={{1,2,3,4},{10,20}}; +packet_size_rate={{1,1,1,1},{2,1}}; + +// Simulation - Don't change + +sim_type = gpgpusim; +//sim_type = latency; +injection_rate = 0.1; + +subnets = 2; + +// Always use read and write no matter following line +//use_read_write = 1; + + +read_request_subnet = 0; +read_reply_subnet = 1; +write_request_subnet = 0; +write_reply_subnet = 1; + +read_request_begin_vc = 0; +read_request_end_vc = 0; +write_request_begin_vc = 0; +write_request_end_vc = 0; +read_reply_begin_vc = 0; +read_reply_end_vc = 0; +write_reply_begin_vc = 0; +write_reply_end_vc = 0; + diff --git a/configs/deprecated-cfgs/GTX480/gpgpusim.config b/configs/deprecated-cfgs/GTX480/gpgpusim.config new file mode 100644 index 0000000..ee90c12 --- /dev/null +++ b/configs/deprecated-cfgs/GTX480/gpgpusim.config @@ -0,0 +1,134 @@ +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 20 + + +# SASS execution (only supported with CUDA >= 4.0) +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 + +# high level architecture configuration +-gpgpu_n_clusters 15 +-gpgpu_n_cores_per_cluster 1 +-gpgpu_n_mem 6 +-gpgpu_n_sub_partition_per_mchannel 2 + +# Fermi clock domains +#-gpgpu_clock_domains ::: +# In Fermi, each pipeline has 16 execution units, so the Core clock needs to be divided +# by 2. (GPGPU-Sim simulates a warp (32 threads) in a single cycle). 1400/2 = 700 +-gpgpu_clock_domains 700.0:700.0:700.0:924.0 + +# shader core pipeline config +-gpgpu_shader_registers 32768 +-gpgpu_occupancy_sm_number 20 + +# This implies a maximum of 48 warps/SM +-gpgpu_shader_core_pipeline 1536:32 +-gpgpu_shader_cta 8 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB +-gpgpu_pipeline_widths 2,1,1,2,1,1,2 +-gpgpu_num_sp_units 2 +-gpgpu_num_sfu_units 1 + +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +-ptx_opcode_latency_int 4,13,4,5,145 +-ptx_opcode_initiation_int 1,2,2,1,8 +-ptx_opcode_latency_fp 4,13,4,5,39 +-ptx_opcode_initiation_fp 1,2,1,1,4 +-ptx_opcode_latency_dp 8,19,8,8,330 +-ptx_opcode_initiation_dp 8,16,8,8,130 + + +# In Fermi, the cache and shared memory can be configured to 16kb:48kb(default) or 48kb:16kb +# ::,::::,::,:** +# ** Optional parameter - Required when mshr_type==Texture Fifo +# Note: Hashing set index function (H) only applies to a set size of 32 or 64. +-gpgpu_cache:dl1 32:128:4,L:L:m:N:H,A:32:8,8 +-gpgpu_shmem_size 49152 + +# The alternative configuration for fermi in case cudaFuncCachePreferL1 is selected +#-gpgpu_cache:dl1 64:128:6,L:L:m:N:H,A:32:8,8 +#-gpgpu_shmem_size 16384 + +# 64 sets, each 128 bytes 8-way for each memory sub partition. This gives 786KB L2 cache +-gpgpu_cache:dl2 64:128:8,L:B:m:W:L,A:32:4,4:0,32 +-gpgpu_cache:dl2_texture_only 0 + +-gpgpu_cache:il1 4:128:4,L:R:f:N:L,A:2:32,4 +-gpgpu_tex_cache:l1 4:128:24,L:R:m:N:L,F:128:4,128:2 +-gpgpu_const_cache:l1 64:64:2,L:R:f:N:L,A:2:32,4 + +# enable operand collector +-gpgpu_operand_collector_num_units_sp 6 +-gpgpu_operand_collector_num_units_sfu 8 +-gpgpu_operand_collector_num_in_ports_sp 2 +-gpgpu_operand_collector_num_out_ports_sp 2 +-gpgpu_num_reg_banks 16 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 + +-gpgpu_max_insn_issue_per_warp 1 + +# interconnection +-network_mode 1 +-inter_config_file config_fermi_islip.icnt + +# memory partition latency config +-rop_latency 120 +-dram_latency 100 + +# dram model config +-gpgpu_dram_scheduler 1 +# The DRAM return queue and the scheduler queue together should provide buffer +# to sustain the memory level parallelism to tolerate DRAM latency +# To allow 100% DRAM utility, there should at least be enough buffer to sustain +# the minimum DRAM latency (100 core cycles). I.e. +# Total buffer space required = 100 x 924MHz / 700MHz = 132 +-gpgpu_frfcfs_dram_sched_queue_size 16 +-gpgpu_dram_return_queue_size 116 + +# for Fermi, bus width is 384bits, this is 8 bytes (4 bytes at each DRAM chip) per memory partition +-gpgpu_n_mem_per_ctrlr 2 +-gpgpu_dram_buswidth 4 +-gpgpu_dram_burst_length 8 +-dram_data_command_freq_ratio 4 # GDDR5 is QDR +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.BBBCCCCB.CCSSSSSS + +# GDDR5 timing from hynix H5GQ1H24AFR +# to disable bank groups, set nbkgrp to 1 and tCCDL and tRTPL to 0 +-gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=6:RCD=12:RAS=28:RP=12:RC=40: + CL=12:WL=4:CDLR=5:WR=12:nbkgrp=4:CCDL=3:RTPL=2" + +# Fermi has two schedulers per core +-gpgpu_num_sched_per_core 2 +# Two Level Scheduler with active and pending pools +#-gpgpu_scheduler two_level_active:6:0:1 +# Loose round robbin scheduler +#-gpgpu_scheduler lrr +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 +-visualizer_enabled 0 + +# power model configs +-power_simulation_enabled 1 +-gpuwattch_xml_file gpuwattch_gtx480.xml + +# tracing functionality +#-trace_enabled 1 +#-trace_components WARP_SCHEDULER,SCOREBOARD +#-trace_sampling_core 0 diff --git a/configs/deprecated-cfgs/GTX480/gpuwattch_gtx480.xml b/configs/deprecated-cfgs/GTX480/gpuwattch_gtx480.xml new file mode 100755 index 0000000..304e0fd --- /dev/null +++ b/configs/deprecated-cfgs/GTX480/gpuwattch_gtx480.xml @@ -0,0 +1,538 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/configs/deprecated-cfgs/GeForceGTX750Ti/config_fermi_islip.icnt b/configs/deprecated-cfgs/GeForceGTX750Ti/config_fermi_islip.icnt new file mode 100644 index 0000000..069ca02 --- /dev/null +++ b/configs/deprecated-cfgs/GeForceGTX750Ti/config_fermi_islip.icnt @@ -0,0 +1,70 @@ +//21*1 fly with 32 flits per packet under gpgpusim injection mode +use_map = 0; +flit_size = 32; + +// currently we do not use this, see subnets below +network_count = 2; + +// Topology +topology = fly; +k = 7; +n = 1; + +// Routing + +routing_function = dest_tag; + +// Flow control + +num_vcs = 1; +vc_buf_size = 8; + +wait_for_tail_credit = 0; + +// Router architecture + +vc_allocator = islip; //separable_input_first; +sw_allocator = islip; //separable_input_first; +alloc_iters = 1; + +credit_delay = 0; +routing_delay = 0; +vc_alloc_delay = 1; +sw_alloc_delay = 1; + +input_speedup = 2; +output_speedup = 1; +internal_speedup = 1.0; + +// Traffic, GPGPU-Sim does not use this + +traffic = uniform; +packet_size ={{1,2,3,4},{10,20}}; +packet_size_rate={{1,1,1,1},{2,1}}; + +// Simulation - Don't change + +sim_type = gpgpusim; +//sim_type = latency; +injection_rate = 0.1; + +subnets = 2; + +// Always use read and write no matter following line +//use_read_write = 1; + + +read_request_subnet = 0; +read_reply_subnet = 1; +write_request_subnet = 0; +write_reply_subnet = 1; + +read_request_begin_vc = 0; +read_request_end_vc = 0; +write_request_begin_vc = 0; +write_request_end_vc = 0; +read_reply_begin_vc = 0; +read_reply_end_vc = 0; +write_reply_begin_vc = 0; +write_reply_end_vc = 0; + diff --git a/configs/deprecated-cfgs/GeForceGTX750Ti/gpgpusim.config b/configs/deprecated-cfgs/GeForceGTX750Ti/gpgpusim.config new file mode 100644 index 0000000..c675aab --- /dev/null +++ b/configs/deprecated-cfgs/GeForceGTX750Ti/gpgpusim.config @@ -0,0 +1,131 @@ +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 52 + + +# SASS execution (only supported with CUDA >= 4.0) +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 + +# high level architecture configuration +-gpgpu_n_clusters 5 +-gpgpu_n_cores_per_cluster 1 +-gpgpu_n_mem 2 +-gpgpu_n_sub_partition_per_mchannel 1 + +# Maxwell clock domains +#-gpgpu_clock_domains ::: +# In Maxwell, each pipeline has 32 execution units, so the Core clock is just the reported number. +-gpgpu_clock_domains 1137.0:1137.0:1137.0:2700.0 + +# shader core pipeline config +-gpgpu_shader_registers 65536 +-gpgpu_occupancy_sm_number 52 + +# This implies a maximum of 64 warps/SM +-gpgpu_shader_core_pipeline 2048:32 +-gpgpu_shader_cta 8 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB +-gpgpu_pipeline_widths 2,1,1,2,1,1,2 +-gpgpu_num_sp_units 8 +-gpgpu_num_sfu_units 32 + +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +-ptx_opcode_latency_int 6,12,13,13,210 +-ptx_opcode_initiation_int 1,1,1,1,4 +-ptx_opcode_latency_fp 6,12,6,6,374 +-ptx_opcode_initiation_fp 1,1,1,1,4 +-ptx_opcode_latency_dp 8,19,8,8,330 +-ptx_opcode_initiation_dp 8,16,8,8,130 + +-gpgpu_cache:dl1 32:128:4,L:L:m:N:H,A:32:8,8 +-gmem_skip_L1D 1 +-gpgpu_shmem_size 65536 + +# The alternative configuration for fermi in case cudaFuncCachePreferL1 is selected +#-gpgpu_cache:dl1 64:128:6,L:L:m:N:H,A:32:8,8 +#-gpgpu_shmem_size 16384 + +# 1024 sets, each 128 bytes 8-way for each memory sub partition. This gives 2MB L2 cache +-gpgpu_cache:dl2 1024:128:8,L:B:m:W:L,A:32:4,4:0,32 +-gpgpu_cache:dl2_texture_only 0 + +-gpgpu_cache:il1 4:128:4,L:R:f:N:L,A:2:32,4 +-gpgpu_tex_cache:l1 16:128:32,L:R:m:N:L,F:128:4,128:2 +-gpgpu_const_cache:l1 64:64:2,L:R:f:N:L,A:2:32,4 + +# enable operand collector +-gpgpu_operand_collector_num_units_sp 6 +-gpgpu_operand_collector_num_units_sfu 8 +-gpgpu_operand_collector_num_in_ports_sp 2 +-gpgpu_operand_collector_num_out_ports_sp 2 +-gpgpu_num_reg_banks 16 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 + +-gpgpu_max_insn_issue_per_warp 2 + +# interconnection +-network_mode 1 +-inter_config_file config_fermi_islip.icnt + +# memory partition latency config +-rop_latency 150 +-dram_latency 130 + +# dram model config +-gpgpu_dram_scheduler 1 +# The DRAM return queue and the scheduler queue together should provide buffer +# to sustain the memory level parallelism to tolerate DRAM latency +# To allow 100% DRAM utility, there should at least be enough buffer to sustain +# the minimum DRAM latency (100 core cycles). I.e. +# Total buffer space required = 100 x 924MHz / 700MHz = 132 +-gpgpu_frfcfs_dram_sched_queue_size 16 +-gpgpu_dram_return_queue_size 300 + +# for Maxwell, bus width is 128bits, this is 64 bytes (32 bytes at each DRAM chip) per memory partition +-gpgpu_n_mem_per_ctrlr 2 +-gpgpu_dram_buswidth 32 +-gpgpu_dram_burst_length 8 +-dram_data_command_freq_ratio 4 # GDDR5 is QDR +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.BBBCCCCB.CCSSSSSS + +# GDDR5 timing from hynix H5GQ1H24AFR +# to disable bank groups, set nbkgrp to 1 and tCCDL and tRTPL to 0 +-gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=6:RCD=12:RAS=28:RP=12:RC=40: + CL=12:WL=4:CDLR=5:WR=12:nbkgrp=4:CCDL=3:RTPL=2" + +# Maxwell has four schedulers per core +-gpgpu_num_sched_per_core 4 +# Two Level Scheduler with active and pending pools +#-gpgpu_scheduler two_level_active:6:0:1 +# Loose round robbin scheduler +#-gpgpu_scheduler lrr +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 +-visualizer_enabled 0 + +# power model configs +# power modeling is currently disabled for GTX750Ti. The gpuwattch_gtx750ti.xml file present is only a placeholder and has NOT been tested for accuracy. +# proper modeling of Maxwell power is planned, but should be considered completely unsupported at present. +-power_simulation_enabled 0 +-gpuwattch_xml_file gpuwattch_gtx750Ti.xml + +# tracing functionality +#-trace_enabled 1 +#-trace_components WARP_SCHEDULER,SCOREBOARD +#-trace_sampling_core 0 diff --git a/configs/deprecated-cfgs/GeForceGTX750Ti/gpuwattch_gtx750Ti.xml b/configs/deprecated-cfgs/GeForceGTX750Ti/gpuwattch_gtx750Ti.xml new file mode 100755 index 0000000..e2b2324 --- /dev/null +++ b/configs/deprecated-cfgs/GeForceGTX750Ti/gpuwattch_gtx750Ti.xml @@ -0,0 +1,539 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/configs/deprecated-cfgs/QuadroFX5600/gpgpusim.config b/configs/deprecated-cfgs/QuadroFX5600/gpgpusim.config new file mode 100644 index 0000000..6f836ee --- /dev/null +++ b/configs/deprecated-cfgs/QuadroFX5600/gpgpusim.config @@ -0,0 +1,99 @@ +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 12 + +# high level architecture configuration +-gpgpu_n_clusters 8 +-gpgpu_n_cores_per_cluster 2 +-gpgpu_n_mem 6 +-gpgpu_clock_domains 337.5:600.0:600.0:800.0 + +# shader core pipeline config +-gpgpu_shader_registers 16384 +-gpgpu_occupancy_sm_number 12 + +-gpgpu_occupancy_sm_number 12 +#8192 (registers per block as written by device Query and which used in this option in our other configurations but this break some benchmarks execution! it does not affect performance modeling though) +-gpgpu_shader_core_pipeline 768:32 +-gpgpu_shader_cta 8 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB +-gpgpu_pipeline_widths 1,1,1,1,1,1,1 +-gpgpu_num_sp_units 1 +-gpgpu_num_sfu_units 1 + +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +-ptx_opcode_latency_int 1,1,19,25,145 +-ptx_opcode_initiation_int 1,1,4,4,32 +-ptx_opcode_latency_fp 1,1,1,1,30 +-ptx_opcode_initiation_fp 1,1,1,1,5 +-ptx_opcode_latency_dp 8,8,8,8,335 +-ptx_opcode_initiation_dp 8,8,8,8,130 + +# memory stage behaviour +-gpgpu_cache:il1 4:256:4,L:R:f:N:L,A:2:32,4 +-gpgpu_tex_cache:l1 8:128:5,L:R:m:N:L,F:128:4,128:2 +-gpgpu_const_cache:l1 64:64:2,L:R:f:N:L,A:2:32,4 +-gpgpu_cache:dl2 16:256:8,L:B:m:W:L,A:16:4,4 +-gpgpu_cache:dl2_texture_only 1 + +# TLB parameters +#-gpgpu_cache:tlbl1 1:524288:16:1:L:R:m,A:32:8,8 +#-gpgpu_tlbl2_latency 45 + + +-gpgpu_shmem_warp_parts 2 + +# interconnection +-network_mode 1 +-inter_config_file icnt_config_islip.icnt + +# dram scheduler config +-gpgpu_dram_scheduler 1 +# The DRAM return queue and the scheduler queue together should provide buffer +# to sustain the memory level parallelism to tolerate DRAM latency +# To allow 100% DRAM utility, there should at least be enough buffer to sustain +# the minimum DRAM latency (30 core cycles). I.e. +# Total buffer space required = 30 x 800MHz / 337.5MHz = 71 +-gpgpu_frfcfs_dram_sched_queue_size 16 +-gpgpu_dram_return_queue_size 55 + +# dram model config +-gpgpu_n_mem_per_ctrlr 2 +-gpgpu_dram_buswidth 4 +-gpgpu_dram_burst_length 4 +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RRBBBCCC.CCCSSSSS +# GDDR3 timing from Samsung K4J52324QH-HC12 @ 800MHz +# {nbk:tCCD:tRRD:tRCD:tRAS:tRP:tRC:CL:WL:tCDLR:tWR:nbkgrp:tCCDL:tRTPL} +-gpgpu_dram_timing_opt nbk=8:CCD=2:RRD=8:RCD=12:RAS=25:RP=10:RC=35:CL=10:WL=7:CDLR=6:WR=11 + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 + + + +# Using cuobjdump to extract ptx/SASS +-gpgpu_ptx_use_cuobjdump 1 + +# SASS execution (only supported with CUDA >= 4.0) +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 + +# enable operand collector +-gpgpu_operand_collector_num_units_sp 6 +-gpgpu_operand_collector_num_units_sfu 8 + +-visualizer_enabled 0 +-power_trace_enabled 0 +-power_simulation_enabled 1 +-gpuwattch_xml_file gpuwattch_quadrofx5600.xml + +-steady_power_levels_enabled 1 +-steady_state_definition 8,4 diff --git a/configs/deprecated-cfgs/QuadroFX5600/gpuwattch_quadrofx5600.xml b/configs/deprecated-cfgs/QuadroFX5600/gpuwattch_quadrofx5600.xml new file mode 100644 index 0000000..2c5a6fc --- /dev/null +++ b/configs/deprecated-cfgs/QuadroFX5600/gpuwattch_quadrofx5600.xml @@ -0,0 +1,538 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/configs/deprecated-cfgs/QuadroFX5600/icnt_config_islip.icnt b/configs/deprecated-cfgs/QuadroFX5600/icnt_config_islip.icnt new file mode 100644 index 0000000..de3bcc8 --- /dev/null +++ b/configs/deprecated-cfgs/QuadroFX5600/icnt_config_islip.icnt @@ -0,0 +1,70 @@ +//14*1 fly with 32 flits per packet under gpgpusim injection mode +use_map = 0; +flit_size = 32; + +// currently we donot use this, see subnets below +network_count = 2; + +// Topology +topology = fly; +k = 14; +n = 1; + +// Routing + +routing_function = dest_tag; + +// Flow control + +num_vcs = 1; +vc_buf_size = 8; + +wait_for_tail_credit = 0; + +// Router architecture + +vc_allocator = islip; //separable_input_first; +sw_allocator = islip; //separable_input_first; +alloc_iters = 1; + +credit_delay = 0; +routing_delay = 0; +vc_alloc_delay = 1; +sw_alloc_delay = 1; + +input_speedup = 2; +output_speedup = 1; +internal_speedup = 1.0; + +// Traffic, GPGPU-Sim does not use this + +traffic = uniform; +packet_size ={{1,2,3,4},{10,20}}; +packet_size_rate={{1,1,1,1},{2,1}}; + +// Simulation - Don't change + +sim_type = gpgpusim; +//sim_type = latency; +injection_rate = 0.1; + +subnets = 2; + +// Always use read and write no matter following line +//use_read_write = 1; + + +read_request_subnet = 0; +read_reply_subnet = 1; +write_request_subnet = 0; +write_reply_subnet = 1; + +read_request_begin_vc = 0; +read_request_end_vc = 0; +write_request_begin_vc = 0; +write_request_end_vc = 0; +read_reply_begin_vc = 0; +read_reply_end_vc = 0; +write_reply_begin_vc = 0; +write_reply_end_vc = 0; + diff --git a/configs/deprecated-cfgs/QuadroFX5800/config_quadro_islip.icnt b/configs/deprecated-cfgs/QuadroFX5800/config_quadro_islip.icnt new file mode 100644 index 0000000..cfe9cac --- /dev/null +++ b/configs/deprecated-cfgs/QuadroFX5800/config_quadro_islip.icnt @@ -0,0 +1,69 @@ +//18*1 fly with 32 flits per packet under gpgpusim injection mode +use_map = 0; +flit_size = 32; + +// currently we donot use this, see subnets below +network_count = 2; + +// Topology +topology = fly; +k = 18; +n = 1; + +// Routing + +routing_function = dest_tag; + +// Flow control + +num_vcs = 1; +vc_buf_size = 8; + +wait_for_tail_credit = 0; + +// Router architecture + +vc_allocator = islip; //separable_input_first; +sw_allocator = islip; //separable_input_first; +alloc_iters = 1; + +credit_delay = 0; +routing_delay = 0; +vc_alloc_delay = 1; +sw_alloc_delay = 1; + +input_speedup = 2; +output_speedup = 1; +internal_speedup = 1.0; + +// Traffic, GPGPU-Sim does not use this +traffic = uniform; +packet_size ={{1,2,3,4},{10,20}}; +packet_size_rate={{1,1,1,1},{2,1}}; + +// Simulation - Don't change +sim_type = gpgpusim; +//sim_type = latency; +injection_rate = 0.1; + + +subnets = 2; + +// Always use read and write no matter following line +//use_read_write = 1; + + +read_request_subnet = 0; +read_reply_subnet = 1; +write_request_subnet = 0; +write_reply_subnet = 1; + +read_request_begin_vc = 0; +read_request_end_vc = 0; +write_request_begin_vc = 0; +write_request_end_vc = 0; +read_reply_begin_vc = 0; +read_reply_end_vc = 0; +write_reply_begin_vc = 0; +write_reply_end_vc = 0; + diff --git a/configs/deprecated-cfgs/QuadroFX5800/gpgpusim.config b/configs/deprecated-cfgs/QuadroFX5800/gpgpusim.config new file mode 100644 index 0000000..fef1110 --- /dev/null +++ b/configs/deprecated-cfgs/QuadroFX5800/gpgpusim.config @@ -0,0 +1,88 @@ +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 13 + +# high level architecture configuration +-gpgpu_n_clusters 10 +-gpgpu_n_cores_per_cluster 3 +-gpgpu_n_mem 8 +-gpgpu_clock_domains 325.0:650.0:650.0:800.0 + +# shader core pipeline config +-gpgpu_shader_registers 16384 +-gpgpu_occupancy_sm_number 13 + +-gpgpu_shader_core_pipeline 1024:32 +-gpgpu_shader_cta 8 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB +-gpgpu_pipeline_widths 1,1,1,1,1,1,1 +-gpgpu_num_sp_units 1 +-gpgpu_num_sfu_units 1 + +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +-ptx_opcode_latency_int 1,1,19,25,145 +-ptx_opcode_initiation_int 1,1,4,4,32 +-ptx_opcode_latency_fp 1,1,1,1,30 +-ptx_opcode_initiation_fp 1,1,1,1,5 +-ptx_opcode_latency_dp 8,8,8,8,335 +-ptx_opcode_initiation_dp 8,8,8,8,130 + +# memory stage behaviour +# ::,::::,::,:** +# ** Optional parameter - Required when mshr_type==Texture Fifo +-gpgpu_cache:il1 4:256:4,L:R:f:N:L,A:2:32,4 +-gpgpu_tex_cache:l1 8:128:5,L:R:m:N:L,F:128:4,128:2 +-gpgpu_const_cache:l1 64:64:2,L:R:f:N:L,A:2:32,4 +-gpgpu_cache:dl2 16:256:8,L:B:m:W:L,A:16:4,4 +-gpgpu_cache:dl2_texture_only 1 + +-gpgpu_shmem_warp_parts 2 + +# interconnection +-network_mode 1 +-inter_config_file config_quadro_islip.icnt + +# dram scheduler config +-gpgpu_dram_scheduler 1 +# The DRAM return queue and the scheduler queue together should provide buffer +# to sustain the memory level parallelism to tolerate DRAM latency +# To allow 100% DRAM utility, there should at least be enough buffer to sustain +# the minimum DRAM latency (30 core cycles). I.e. +# Total buffer space required = 30 x 800MHz / 325MHz = 74 +-gpgpu_frfcfs_dram_sched_queue_size 16 +-gpgpu_dram_return_queue_size 58 + +# dram model config +-gpgpu_n_mem_per_ctrlr 2 +-gpgpu_dram_buswidth 4 +-gpgpu_dram_burst_length 4 +-dram_data_command_freq_ratio 2 # GDDR3 is DDR +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RRBBBCCC.CCCSSSSS +# GDDR3 timing from Samsung K4J52324QH-HC12 @ 800MHz +-gpgpu_dram_timing_opt nbk=8:CCD=2:RRD=8:RCD=12:RAS=25:RP=10:RC=35:CL=10:WL=7:CDLR=6:WR=11 + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 + +# SASS execution (only supported with CUDA >= 4.0) +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 + +# enable operand collector +-gpgpu_operand_collector_num_units_sp 6 +-gpgpu_operand_collector_num_units_sfu 8 + +-visualizer_enabled 0 + +# tracing functionality +#-trace_enabled 1 +#-trace_components WARP_SCHEDULER,SCOREBOARD +#-trace_sampling_core 0 diff --git a/configs/deprecated-cfgs/SM6_GTX1080/config_fermi_islip.icnt b/configs/deprecated-cfgs/SM6_GTX1080/config_fermi_islip.icnt new file mode 100644 index 0000000..2a69ddd --- /dev/null +++ b/configs/deprecated-cfgs/SM6_GTX1080/config_fermi_islip.icnt @@ -0,0 +1,70 @@ +//21*1 fly with 32 flits per packet under gpgpusim injection mode +use_map = 0; +flit_size = 32; + +// currently we do not use this, see subnets below +network_count = 2; + +// Topology +topology = fly; +k = 50; +n = 1; + +// Routing + +routing_function = dest_tag; + +// Flow control + +num_vcs = 1; +vc_buf_size = 8; + +wait_for_tail_credit = 0; + +// Router architecture + +vc_allocator = islip; //separable_input_first; +sw_allocator = islip; //separable_input_first; +alloc_iters = 1; + +credit_delay = 0; +routing_delay = 0; +vc_alloc_delay = 1; +sw_alloc_delay = 1; + +input_speedup = 2; +output_speedup = 1; +internal_speedup = 1.0; + +// Traffic, GPGPU-Sim does not use this + +traffic = uniform; +packet_size ={{1,2,3,4},{10,20}}; +packet_size_rate={{1,1,1,1},{2,1}}; + +// Simulation - Don't change + +sim_type = gpgpusim; +//sim_type = latency; +injection_rate = 0.1; + +subnets = 2; + +// Always use read and write no matter following line +//use_read_write = 1; + + +read_request_subnet = 0; +read_reply_subnet = 1; +write_request_subnet = 0; +write_reply_subnet = 1; + +read_request_begin_vc = 0; +read_request_end_vc = 0; +write_request_begin_vc = 0; +write_request_end_vc = 0; +read_reply_begin_vc = 0; +read_reply_end_vc = 0; +write_reply_begin_vc = 0; +write_reply_end_vc = 0; + diff --git a/configs/deprecated-cfgs/SM6_GTX1080/gpgpusim.config b/configs/deprecated-cfgs/SM6_GTX1080/gpgpusim.config new file mode 100644 index 0000000..fb044c6 --- /dev/null +++ b/configs/deprecated-cfgs/SM6_GTX1080/gpgpusim.config @@ -0,0 +1,150 @@ +# This config models the Pascal GP102 (GeForceGTX 1080Ti) + +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 60 + +# SASS execution (only supported with CUDA >= 4.0) +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 + +# high level architecture configuration +-gpgpu_n_clusters 28 +-gpgpu_n_cores_per_cluster 1 +-gpgpu_n_mem 11 +-gpgpu_n_sub_partition_per_mchannel 2 + +# Pascal clock domains +#-gpgpu_clock_domains ::: +# Pascal NVIDIA TITAN X clock domains are adopted from +# https://en.wikipedia.org/wiki/GeForce_10_series +-gpgpu_clock_domains 1481.0:2962.0:1481.0:2750.0 + +# shader core pipeline config +-gpgpu_shader_registers 65536 +-gpgpu_occupancy_sm_number 60 + +# This implies a maximum of 64 warps/SM +-gpgpu_shader_core_pipeline 2048:32 +-gpgpu_shader_cta 32 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB +## Pascal GP102 has 4 SP SIMD units and 1 SFU unit +## we need to scale the number of pipeline registers to be equal to the number of SP units +-gpgpu_pipeline_widths 4,1,1,4,1,1,6 +-gpgpu_num_sp_units 4 +-gpgpu_num_sfu_units 1 + +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +# SFU is 32-width in pascal, then dp units initiation is 1 cycle +-ptx_opcode_latency_int 4,13,4,5,145 +-ptx_opcode_initiation_int 1,2,2,2,8 +-ptx_opcode_latency_fp 4,13,4,5,39 +-ptx_opcode_initiation_fp 1,2,1,1,4 +-ptx_opcode_latency_dp 8,19,8,8,330 +-ptx_opcode_initiation_dp 1,2,1,1,130 + +# ::,::::,::,:** +# ** Optional parameter - Required when mshr_type==Texture Fifo +# Note: Hashing set index function (H) only applies to a set size of 32 or 64. +# Pascal GP102 has 96KB Shared memory +# Pascal GP102 has 64KB L1 cache +# The default is to disable the L1 cache, unless cache modifieres is used +-gpgpu_cache:dl1 64:128:6,L:L:m:N:H,A:128:8,8 +-gpgpu_shmem_size 98304 +-gmem_skip_L1D 1 + +# 64 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 3MB L2 cache +-gpgpu_cache:dl2 64:128:16,L:B:m:W:L,A:1024:1024,4:0,32 # used to be 128:4 +-gpgpu_cache:dl2_texture_only 0 + +# 4 KB Inst. +-gpgpu_cache:il1 8:128:4,L:R:f:N:L,A:2:48,4 +# 48 KB Tex +-gpgpu_tex_cache:l1 16:128:24,L:R:m:N:L,F:128:4,128:2 +# 12 KB Const +-gpgpu_const_cache:l1 128:64:2,L:R:f:N:L,A:2:64,4 + +# enable operand collector +## larger operand collectors and reg_banks are needed for the 4 warp schedulers and 4 SIMD units +-gpgpu_operand_collector_num_units_sp 20 +-gpgpu_operand_collector_num_units_sfu 4 +-gpgpu_operand_collector_num_units_mem 8 +-gpgpu_operand_collector_num_in_ports_sp 4 +-gpgpu_operand_collector_num_out_ports_sp 4 +-gpgpu_operand_collector_num_in_ports_sfu 1 +-gpgpu_operand_collector_num_out_ports_sfu 1 +-gpgpu_operand_collector_num_in_ports_mem 1 +-gpgpu_operand_collector_num_out_ports_mem 1 +# gpgpu_num_reg_banks should be increased to 32, but it gives an error! +-gpgpu_num_reg_banks 32 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 + +## In Pascal, a warp scheduler can issue 2 insts per cycle +-gpgpu_max_insn_issue_per_warp 2 + +# interconnection +-network_mode 1 +-inter_config_file config_fermi_islip.icnt + +# memory partition latency config +-rop_latency 120 +-dram_latency 100 + +# dram model config +-gpgpu_dram_scheduler 1 +# The DRAM return queue and the scheduler queue together should provide buffer +# to sustain the memory level parallelism to tolerate DRAM latency +# To allow 100% DRAM utility, there should at least be enough buffer to sustain +# the minimum DRAM latency (100 core cycles). I.e. +# Total buffer space required = 100 x 924MHz / 700MHz = 132 +-gpgpu_frfcfs_dram_sched_queue_size 64 +-gpgpu_dram_return_queue_size 116 + +# for NVIDIA GeForceGTX 1080Ti, bus width is 352bits (11 DRAM chips x 32 bits) +# 11 memory paritions, 4 bytes (1 DRAM chip) per memory partition +# the atom size of GDDR5X (the smallest read request) is 32 bytes +-gpgpu_n_mem_per_ctrlr 1 +-gpgpu_dram_buswidth 4 +-gpgpu_dram_burst_length 8 +-dram_data_command_freq_ratio 4 # GDDR5X is QDR +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS + +# Use the same GDDR5 timing from hynix H5GQ1H24AFR +# disable bank groups for now, set nbkgrp to 1 and tCCDL and tRTPL to 0 +-gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=6:RCD=12:RAS=28:RP=12:RC=40: + CL=12:WL=4:CDLR=5:WR=12:nbkgrp=1:CCDL=0:RTPL=0" + +# Pascal has four schedulers per core +-gpgpu_num_sched_per_core 2 +# Two Level Scheduler with active and pending pools +#-gpgpu_scheduler two_level_active:6:0:1 +# Loose round robbin scheduler +#-gpgpu_scheduler lrr +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 +-visualizer_enabled 0 + +# power model configs +-power_simulation_enabled 1 +-gpuwattch_xml_file gpuwattch_gtx1080Ti.xml + +# tracing functionality +#-trace_enabled 1 +#-trace_components WARP_SCHEDULER,SCOREBOARD +#-trace_sampling_core 0 + diff --git a/configs/deprecated-cfgs/SM6_GTX1080/gpuwattch_gtx1080Ti.xml b/configs/deprecated-cfgs/SM6_GTX1080/gpuwattch_gtx1080Ti.xml new file mode 100755 index 0000000..02619ff --- /dev/null +++ b/configs/deprecated-cfgs/SM6_GTX1080/gpuwattch_gtx1080Ti.xml @@ -0,0 +1,538 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/configs/deprecated-cfgs/SM6_P100/config_fermi_islip.icnt b/configs/deprecated-cfgs/SM6_P100/config_fermi_islip.icnt new file mode 100644 index 0000000..d26c8d9 --- /dev/null +++ b/configs/deprecated-cfgs/SM6_P100/config_fermi_islip.icnt @@ -0,0 +1,70 @@ +//21*1 fly with 32 flits per packet under gpgpusim injection mode +use_map = 0; +flit_size = 32; + +// currently we do not use this, see subnets below +network_count = 2; + +// Topology +topology = fly; +k = 60; +n = 1; + +// Routing + +routing_function = dest_tag; + +// Flow control + +num_vcs = 1; +vc_buf_size = 8; + +wait_for_tail_credit = 0; + +// Router architecture + +vc_allocator = islip; //separable_input_first; +sw_allocator = islip; //separable_input_first; +alloc_iters = 1; + +credit_delay = 0; +routing_delay = 0; +vc_alloc_delay = 1; +sw_alloc_delay = 1; + +input_speedup = 2; +output_speedup = 1; +internal_speedup = 1.0; + +// Traffic, GPGPU-Sim does not use this + +traffic = uniform; +packet_size ={{1,2,3,4},{10,20}}; +packet_size_rate={{1,1,1,1},{2,1}}; + +// Simulation - Don't change + +sim_type = gpgpusim; +//sim_type = latency; +injection_rate = 0.1; + +subnets = 2; + +// Always use read and write no matter following line +//use_read_write = 1; + + +read_request_subnet = 0; +read_reply_subnet = 1; +write_request_subnet = 0; +write_reply_subnet = 1; + +read_request_begin_vc = 0; +read_request_end_vc = 0; +write_request_begin_vc = 0; +write_request_end_vc = 0; +read_reply_begin_vc = 0; +read_reply_end_vc = 0; +write_reply_begin_vc = 0; +write_reply_end_vc = 0; + diff --git a/configs/deprecated-cfgs/SM6_P100/gpgpusim.config b/configs/deprecated-cfgs/SM6_P100/gpgpusim.config new file mode 100644 index 0000000..a5e6736 --- /dev/null +++ b/configs/deprecated-cfgs/SM6_P100/gpgpusim.config @@ -0,0 +1,156 @@ +# This config models the Pascal GP100 +# For more info about this card, see Nvidia White paper +# https://images.nvidia.com/content/pdf/tesla/whitepaper/pascal-architecture-whitepaper.pdf + +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 60 + +# SASS execution (only supported with CUDA >= 4.0) +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 + +# high level architecture configuration +-gpgpu_n_clusters 28 +-gpgpu_n_cores_per_cluster 2 +-gpgpu_n_mem 32 +-gpgpu_n_sub_partition_per_mchannel 1 + +# Pscal clock domains +#-gpgpu_clock_domains ::: +# Pascal NVIDIA GP100 clock domains are adopted from +# https://en.wikipedia.org/wiki/Nvidia_Tesla +-gpgpu_clock_domains 1480.0:1480.0:1480.0:715.0 + +# shader core pipeline config +-gpgpu_shader_registers 65536 + +# This implies a maximum of 64 warps/SM +-gpgpu_shader_core_pipeline 2048:32 +-gpgpu_shader_cta 32 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB +## Pascal GP102 has 4 SP SIMD units and 1 SFU unit +## we need to scale the number of pipeline registers to be equal to the number of SP units +-gpgpu_pipeline_widths 2,2,1,2,2,1,4 +-gpgpu_num_sp_units 2 +-gpgpu_num_sfu_units 1 + +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +# SFU is 32-width in pascal, then dp units initiation is 1 cycle +-ptx_opcode_latency_int 4,13,4,5,145 +-ptx_opcode_initiation_int 1,1,1,1,4 +-ptx_opcode_latency_fp 4,13,4,5,39 +-ptx_opcode_initiation_fp 1,2,1,1,4 +-ptx_opcode_latency_dp 8,19,8,8,330 +-ptx_opcode_initiation_dp 2,2,2,2,130 + +# ::,::::,::,:** +# ** Optional parameter - Required when mshr_type==Texture Fifo +# Note: Hashing set index function (H) only applies to a set size of 32 or 64. +# Pascal GP100 has 64KB Shared memory +# Pascal GP100 has 48KB L1 cache +-gpgpu_cache:dl1 64:128:6,L:L:m:N:H,A:128:8,16 +-gpgpu_cache:dl1PrefL1 64:128:6,L:L:m:N:H,A:128:8,16 +-gpgpu_cache:dl1PrefShared 64:128:6,L:L:m:N:H,A:128:8,16 +-gpgpu_shmem_size 65536 +-gpgpu_shmem_size_PrefL1 65536 +-gpgpu_shmem_size_PrefShared 65536 +-gmem_skip_L1D 0 + +# 64 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 3MB L2 cache +-gpgpu_cache:dl2 64:128:16,L:B:m:W:L,A:128:8,4:0,32 +-gpgpu_cache:dl2_texture_only 0 +-gpgpu_dram_partition_queues 32:32:32:32 + +# 4 KB Inst. +-gpgpu_cache:il1 8:128:4,L:R:f:N:L,A:2:48,4 +# 48 KB Tex +-gpgpu_tex_cache:l1 16:128:24,L:R:m:N:L,F:128:4,128:2 +# 12 KB Const +-gpgpu_const_cache:l1 128:64:2,L:R:f:N:L,A:2:64,4 + +# enable operand collector +-gpgpu_operand_collector_num_units_sp 14 +-gpgpu_operand_collector_num_units_sfu 8 +-gpgpu_operand_collector_num_units_mem 10 +-gpgpu_operand_collector_num_in_ports_sp 4 +-gpgpu_operand_collector_num_out_ports_sp 4 +-gpgpu_operand_collector_num_in_ports_sfu 1 +-gpgpu_operand_collector_num_out_ports_sfu 1 +-gpgpu_operand_collector_num_in_ports_mem 1 +-gpgpu_operand_collector_num_out_ports_mem 1 +-gpgpu_num_reg_banks 32 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 + +## In Pascal, a warp scheduler can issue 2 insts per cycle +-gpgpu_max_insn_issue_per_warp 2 + +# interconnection +-network_mode 1 +-inter_config_file config_fermi_islip.icnt + +# memory partition latency config +-rop_latency 120 +-dram_latency 100 + +# dram model config +-gpgpu_dram_scheduler 1 +# The DRAM return queue and the scheduler queue together should provide buffer +# to sustain the memory level parallelism to tolerate DRAM latency +# To allow 100% DRAM utility, there should at least be enough buffer to sustain +# the minimum DRAM latency (100 core cycles). I.e. +# Total buffer space required = 100 x 924MHz / 700MHz = 132 +-gpgpu_frfcfs_dram_sched_queue_size 64 +-gpgpu_dram_return_queue_size 116 + +# for HBM, 32 channles, each (128 bits) 16 bytes width +-gpgpu_n_mem_per_ctrlr 1 +-gpgpu_dram_buswidth 16 +-gpgpu_dram_burst_length 2 +-dram_data_command_freq_ratio 2 # HBM is DDR +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBBCCC.CCCSSSSS + +# HBM timing are adopted from hynix JESD235 standered and nVidia HPCA 2017 paper (http://www.cs.utah.edu/~nil/pubs/hpca17.pdf) +# Timing for 1 GHZ +# tRRDl and tWTR are missing, need to be added +#-gpgpu_dram_timing_opt "nbk=16:CCD=1:RRD=4:RCD=14:RAS=33:RP=14:RC=47: +# CL=14:WL=2:CDLR=3:WR=12:nbkgrp=4:CCDL=2:RTPL=4" + +# Timing for 715 MHZ, Tesla Pascal P100 HBM runs at 715 MHZ +-gpgpu_dram_timing_opt "nbk=16:CCD=1:RRD=3:RCD=10:RAS=24:RP=10:RC=34: + CL=10:WL=2:CDLR=3:WR=9:nbkgrp=4:CCDL=2:RTPL=3" + +# Fermi has two schedulers per core +-gpgpu_num_sched_per_core 2 +# Two Level Scheduler with active and pending pools +#-gpgpu_scheduler two_level_active:6:0:1 +# Loose round robbin scheduler +#-gpgpu_scheduler lrr +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 +-visualizer_enabled 0 + +# power model configs +-power_simulation_enabled 0 +-gpuwattch_xml_file gpuwattch_gtx480.xml + +# tracing functionality +#-trace_enabled 1 +#-trace_components WARP_SCHEDULER,SCOREBOARD +#-trace_sampling_core 0 + diff --git a/configs/deprecated-cfgs/SM6_P100/gpuwattch_gtx480.xml b/configs/deprecated-cfgs/SM6_P100/gpuwattch_gtx480.xml new file mode 100755 index 0000000..304e0fd --- /dev/null +++ b/configs/deprecated-cfgs/SM6_P100/gpuwattch_gtx480.xml @@ -0,0 +1,538 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/configs/deprecated-cfgs/SM6_TITANX/config_fermi_islip.icnt b/configs/deprecated-cfgs/SM6_TITANX/config_fermi_islip.icnt new file mode 100644 index 0000000..602daee --- /dev/null +++ b/configs/deprecated-cfgs/SM6_TITANX/config_fermi_islip.icnt @@ -0,0 +1,70 @@ +//21*1 fly with 32 flits per packet under gpgpusim injection mode +use_map = 0; +flit_size = 32; + +// currently we do not use this, see subnets below +network_count = 2; + +// Topology +topology = fly; +k = 52; +n = 1; + +// Routing + +routing_function = dest_tag; + +// Flow control + +num_vcs = 1; +vc_buf_size = 8; + +wait_for_tail_credit = 0; + +// Router architecture + +vc_allocator = islip; //separable_input_first; +sw_allocator = islip; //separable_input_first; +alloc_iters = 1; + +credit_delay = 0; +routing_delay = 0; +vc_alloc_delay = 1; +sw_alloc_delay = 1; + +input_speedup = 2; +output_speedup = 1; +internal_speedup = 1.0; + +// Traffic, GPGPU-Sim does not use this + +traffic = uniform; +packet_size ={{1,2,3,4},{10,20}}; +packet_size_rate={{1,1,1,1},{2,1}}; + +// Simulation - Don't change + +sim_type = gpgpusim; +//sim_type = latency; +injection_rate = 0.1; + +subnets = 2; + +// Always use read and write no matter following line +//use_read_write = 1; + + +read_request_subnet = 0; +read_reply_subnet = 1; +write_request_subnet = 0; +write_reply_subnet = 1; + +read_request_begin_vc = 0; +read_request_end_vc = 0; +write_request_begin_vc = 0; +write_request_end_vc = 0; +read_reply_begin_vc = 0; +read_reply_end_vc = 0; +write_reply_begin_vc = 0; +write_reply_end_vc = 0; + diff --git a/configs/deprecated-cfgs/SM6_TITANX/gpgpusim.config b/configs/deprecated-cfgs/SM6_TITANX/gpgpusim.config new file mode 100644 index 0000000..28912a3 --- /dev/null +++ b/configs/deprecated-cfgs/SM6_TITANX/gpgpusim.config @@ -0,0 +1,157 @@ +# This config models the Pascal GP102 (NVIDIA TITAN X) +# For more info about this card, see Nvidia White paper +# http://international.download.nvidia.com/geforce-com/international/pdfs/GeForce_GTX_1080_Whitepaper_FINAL.pdf + +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 61 + +# SASS execution (only supported with CUDA >= 4.0) +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 + +# high level architecture configuration +-gpgpu_n_clusters 28 +-gpgpu_n_cores_per_cluster 1 +-gpgpu_n_mem 12 +-gpgpu_n_sub_partition_per_mchannel 2 + +# Pascal clock domains +#-gpgpu_clock_domains ::: +# Pascal NVIDIA TITAN X clock domains are adopted from +# https://en.wikipedia.org/wiki/GeForce_10_series +-gpgpu_clock_domains 1417.0:1417.0:1417.0:2500.0 + +# shader core pipeline config +-gpgpu_shader_registers 65536 +-gpgpu_occupancy_sm_number 61 + +# This implies a maximum of 64 warps/SM +-gpgpu_shader_core_pipeline 2048:32 +-gpgpu_shader_cta 32 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB +## Pascal GP102 has 4 SP SIMD units and 1 SFU unit +## we need to scale the number of pipeline registers to be equal to the number of SP units +-gpgpu_pipeline_widths 4,1,1,4,1,1,6 +-gpgpu_num_sp_units 4 +-gpgpu_num_sfu_units 1 + +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +# SFU is 32-width in pascal, then dp units initiation is 1 cycle +-ptx_opcode_latency_int 4,13,4,5,145 +-ptx_opcode_initiation_int 1,1,1,1,4 +-ptx_opcode_latency_fp 4,13,4,5,39 +-ptx_opcode_initiation_fp 1,2,1,1,4 +-ptx_opcode_latency_dp 8,19,8,8,330 +-ptx_opcode_initiation_dp 4,8,4,4,130 + +# ::,::::,::,:** +# ** Optional parameter - Required when mshr_type==Texture Fifo +# Note: Hashing set index function (H) only applies to a set size of 32 or 64. +# Pascal GP102 has 96KB Shared memory +# Pascal GP102 has 64KB L1 cache +# The defulat is to disable the L1 cache, unless cache modifieres is used +-gpgpu_cache:dl1 64:128:6,L:L:m:N:H,A:128:8,16 +-gpgpu_cache:dl1PrefL1 64:128:6,L:L:m:N:H,A:128:8,16 +-gpgpu_cache:dl1PrefShared 64:128:6,L:L:m:N:H,A:128:8,16 +-gpgpu_shmem_size 98304 +-gpgpu_shmem_size_PrefL1 98304 +-gpgpu_shmem_size_PrefShared 98304 +-gmem_skip_L1D 1 + +# 64 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 3MB L2 cache +-gpgpu_cache:dl2 64:128:16,L:B:m:W:L,A:128:8,4:0,32 +-gpgpu_cache:dl2_texture_only 0 +-gpgpu_dram_partition_queues 32:32:32:32 + +# 4 KB Inst. +-gpgpu_cache:il1 8:128:4,L:R:f:N:L,A:2:48,4 +# 48 KB Tex +-gpgpu_tex_cache:l1 16:128:24,L:R:m:N:L,F:128:4,128:2 +# 12 KB Const +-gpgpu_const_cache:l1 128:64:2,L:R:f:N:L,A:2:64,4 + +# enable operand collector +## larger operand collectors and reg_banks are needed for the 4 warp schedulers and 4 SIMD units +-gpgpu_operand_collector_num_units_sp 20 +-gpgpu_operand_collector_num_units_sfu 4 +-gpgpu_operand_collector_num_units_mem 8 +-gpgpu_operand_collector_num_in_ports_sp 4 +-gpgpu_operand_collector_num_out_ports_sp 4 +-gpgpu_operand_collector_num_in_ports_sfu 1 +-gpgpu_operand_collector_num_out_ports_sfu 1 +-gpgpu_operand_collector_num_in_ports_mem 1 +-gpgpu_operand_collector_num_out_ports_mem 1 +# gpgpu_num_reg_banks should be increased to 32 +-gpgpu_num_reg_banks 32 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 + +## In Pascal, a warp scheduler can issue 2 insts per cycle +-gpgpu_max_insn_issue_per_warp 2 + +# interconnection +-network_mode 1 +-inter_config_file config_fermi_islip.icnt + +# memory partition latency config +-rop_latency 120 +-dram_latency 100 + +# dram model config +-gpgpu_dram_scheduler 1 +# The DRAM return queue and the scheduler queue together should provide buffer +# to sustain the memory level parallelism to tolerate DRAM latency +# To allow 100% DRAM utility, there should at least be enough buffer to sustain +# the minimum DRAM latency (100 core cycles). I.e. +# Total buffer space required = 100 x 924MHz / 700MHz = 132 +-gpgpu_frfcfs_dram_sched_queue_size 64 +-gpgpu_dram_return_queue_size 116 + +# for NVIDIA TITAN X, bus width is 384bits (12 DRAM chips x 32 bits) +# 12 memory paritions, 4 bytes (1 DRAM chip) per memory partition +# the atom size of GDDR5X (the smallest read request) is 32 bytes +-gpgpu_n_mem_per_ctrlr 1 +-gpgpu_dram_buswidth 4 +-gpgpu_dram_burst_length 8 +-dram_data_command_freq_ratio 4 # GDDR5X is QDR +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS + +# Use the same GDDR5 timing from hynix H5GQ1H24AFR +# disable bank groups for now, set nbkgrp to 1 and tCCDL and tRTPL to 0 +-gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=6:RCD=12:RAS=28:RP=12:RC=40: + CL=12:WL=4:CDLR=5:WR=12:nbkgrp=1:CCDL=0:RTPL=0" + +# Fermi has two schedulers per core +-gpgpu_num_sched_per_core 4 +# Two Level Scheduler with active and pending pools +#-gpgpu_scheduler two_level_active:6:0:1 +# Loose round robbin scheduler +#-gpgpu_scheduler lrr +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 +-visualizer_enabled 0 + +# power model configs +-power_simulation_enabled 1 +-gpuwattch_xml_file gpuwattch_gtx480.xml + +# tracing functionality +#-trace_enabled 1 +#-trace_components WARP_SCHEDULER,SCOREBOARD +#-trace_sampling_core 0 + diff --git a/configs/deprecated-cfgs/SM6_TITANX/gpuwattch_gtx480.xml b/configs/deprecated-cfgs/SM6_TITANX/gpuwattch_gtx480.xml new file mode 100755 index 0000000..304e0fd --- /dev/null +++ b/configs/deprecated-cfgs/SM6_TITANX/gpuwattch_gtx480.xml @@ -0,0 +1,538 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/configs/deprecated-cfgs/SM7_TITANV/config_fermi_islip.icnt b/configs/deprecated-cfgs/SM7_TITANV/config_fermi_islip.icnt new file mode 100644 index 0000000..fac792a --- /dev/null +++ b/configs/deprecated-cfgs/SM7_TITANV/config_fermi_islip.icnt @@ -0,0 +1,70 @@ +//21*1 fly with 32 flits per packet under gpgpusim injection mode +use_map = 0; +flit_size = 32; + +// currently we do not use this, see subnets below +network_count = 2; + +// Topology +topology = fly; +k = 64; +n = 1; + +// Routing + +routing_function = dest_tag; + +// Flow control + +num_vcs = 1; +vc_buf_size = 8; + +wait_for_tail_credit = 0; + +// Router architecture + +vc_allocator = islip; //separable_input_first; +sw_allocator = islip; //separable_input_first; +alloc_iters = 1; + +credit_delay = 0; +routing_delay = 0; +vc_alloc_delay = 1; +sw_alloc_delay = 1; + +input_speedup = 2; +output_speedup = 1; +internal_speedup = 1.0; + +// Traffic, GPGPU-Sim does not use this + +traffic = uniform; +packet_size ={{1,2,3,4},{10,20}}; +packet_size_rate={{1,1,1,1},{2,1}}; + +// Simulation - Don't change + +sim_type = gpgpusim; +//sim_type = latency; +injection_rate = 0.1; + +subnets = 2; + +// Always use read and write no matter following line +//use_read_write = 1; + + +read_request_subnet = 0; +read_reply_subnet = 1; +write_request_subnet = 0; +write_reply_subnet = 1; + +read_request_begin_vc = 0; +read_request_end_vc = 0; +write_request_begin_vc = 0; +write_request_end_vc = 0; +read_reply_begin_vc = 0; +read_reply_end_vc = 0; +write_reply_begin_vc = 0; +write_reply_end_vc = 0; + diff --git a/configs/deprecated-cfgs/SM7_TITANV/gpgpusim.config b/configs/deprecated-cfgs/SM7_TITANV/gpgpusim.config new file mode 100644 index 0000000..aefb04a --- /dev/null +++ b/configs/deprecated-cfgs/SM7_TITANV/gpgpusim.config @@ -0,0 +1,165 @@ +# This config models the Volta Titan X +# For more info about this card: +# http://images.nvidia.com/content/volta-architecture/pdf/volta-architecture-whitepaper.pdf +# https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=8344474&tag=1# +# http://on-demand.gputechconf.com/gtc/2018/presentation/s8122-dissecting-the-volta-gpu-architecture-through-microbenchmarking.pdf +# https://en.wikipedia.org/wiki/Volta_(microarchitecture) +# https://www.hotchips.org/wp-content/uploads/hc_archives/hc29/HC29.21-Monday-Pub/HC29.21.10-GPU-Gaming-Pub/HC29.21.132-Volta-Choquette-NVIDIA-Final3.pdf +# https://devblogs.nvidia.com/inside-volta/ +# http://on-demand.gputechconf.com/gtc/2017/presentation/s7798-luke-durant-inside-volta.pdf + +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 70 + +# SASS execution (only supported with CUDA >= 4.0) +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 + +# high level architecture configuration +-gpgpu_n_clusters 40 +-gpgpu_n_cores_per_cluster 2 +-gpgpu_n_mem 24 +-gpgpu_n_sub_partition_per_mchannel 1 + +# volta clock domains +#-gpgpu_clock_domains ::: +# Pascal NVIDIA GP100 clock domains are adopted from +# https://en.wikipedia.org/wiki/Volta_(microarchitecture) +-gpgpu_clock_domains 1200.0:1200.0:1200.0:850.0 +# boost mode +# -gpgpu_clock_domains 1455.0:2000.0:1455.0:850.0 + +# shader core pipeline config +-gpgpu_shader_registers 65536 + +# This implies a maximum of 64 warps/SM +-gpgpu_shader_core_pipeline 2048:32 +-gpgpu_shader_cta 32 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB +## Pascal GP100 has 2 SP SIMD units, 2 SFU units, 2 DP units per core +## we need to scale the number of pipeline registers to be equal to the number of SP units +-gpgpu_pipeline_widths 4,4,1,4,4,1,9 +-gpgpu_num_sp_units 4 +-gpgpu_num_sfu_units 4 + +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +# All Div operations are executed on SFU unit +# Throughput (initiation latency) are adopted from +# http://on-demand.gputechconf.com/gtc/2018/presentation/s8122-dissecting-the-volta-gpu-architecture-through-microbenchmarking.pdf +-ptx_opcode_latency_int 4,13,4,5,145 +-ptx_opcode_initiation_int 2,2,2,2,8 +-ptx_opcode_latency_fp 4,13,4,5,39 +-ptx_opcode_initiation_fp 2,2,2,2,4 +-ptx_opcode_latency_dp 8,19,8,8,330 +-ptx_opcode_initiation_dp 4,4,4,4,130 + +# ::,::::,::,:** +# ** Optional parameter - Required when mshr_type==Texture Fifo +# Note: Hashing set index function (H) only applies to a set size of 32 or 64. +# Volta GV100 has 64KB Shared memory +-gpgpu_cache:dl1 64:128:8,L:L:m:N:H,A:256:8,16:0 +-gpgpu_cache:dl1PrefL1 64:128:16,L:L:m:N:H,A:256:8,16:0 +-gpgpu_cache:dl1PrefShared 32:128:6,L:L:m:N:H,A:256:8,16:0 +-gpgpu_shmem_size 65536 +-gpgpu_shmem_size_PrefL1 1 +-gpgpu_shmem_size_PrefShared 98304 +-gmem_skip_L1D 0 + +# 64 sets, each 128 bytes 24-way for each memory sub partition (192 KB per memory sub partition). This gives 4.5MB L2 cache +-gpgpu_cache:dl2 64:128:24,L:B:m:W:L,A:256:4,32:0,32 +-gpgpu_cache:dl2_texture_only 0 +-gpgpu_dram_partition_queues 64:64:64:64 + +# 128 KB Inst. +-gpgpu_cache:il1 64:128:16,L:R:f:N:L,A:2:48,4 +# 48 KB Tex +-gpgpu_tex_cache:l1 16:128:24,L:R:m:N:L,F:128:4,128:2 +# 64 KB Const +-gpgpu_const_cache:l1 128:64:8,L:R:f:N:L,A:2:64,4 + +# enable operand collector +-gpgpu_operand_collector_num_units_sp 14 +-gpgpu_operand_collector_num_units_sfu 8 +-gpgpu_operand_collector_num_units_mem 10 +-gpgpu_operand_collector_num_in_ports_sp 4 +-gpgpu_operand_collector_num_out_ports_sp 4 +-gpgpu_operand_collector_num_in_ports_sfu 1 +-gpgpu_operand_collector_num_out_ports_sfu 1 +-gpgpu_operand_collector_num_in_ports_mem 1 +-gpgpu_operand_collector_num_out_ports_mem 1 +-gpgpu_num_reg_banks 32 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 + +## In Pascal, a warp scheduler can issue 2 insts per cycle +-gpgpu_max_insn_issue_per_warp 1 + +# interconnection +-network_mode 1 +-inter_config_file config_fermi_islip.icnt + +# memory partition latency config +-rop_latency 120 +-dram_latency 100 + +# dram model config +-gpgpu_dram_scheduler 1 +# The DRAM return queue and the scheduler queue together should provide buffer +# to sustain the memory level parallelism to tolerate DRAM latency +# To allow 100% DRAM utility, there should at least be enough buffer to sustain +# the minimum DRAM latency (100 core cycles). I.e. +# Total buffer space required = 100 x 924MHz / 700MHz = 132 +-gpgpu_frfcfs_dram_sched_queue_size 64 +-gpgpu_dram_return_queue_size 116 + +# for HBM, 32 channles, each (128 bits) 16 bytes width +-gpgpu_n_mem_per_ctrlr 1 +-gpgpu_dram_buswidth 16 +-gpgpu_dram_burst_length 2 +-dram_data_command_freq_ratio 2 # HBM is DDR +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBBCCC.CCCSSSSS + +# HBM timing are adopted from hynix JESD235 standered and nVidia HPCA 2017 paper (http://www.cs.utah.edu/~nil/pubs/hpca17.pdf) +# Timing for 1 GHZ +# tRRDl and tWTR are missing, need to be added +#-gpgpu_dram_timing_opt "nbk=16:CCD=1:RRD=4:RCD=14:RAS=33:RP=14:RC=47: +# CL=14:WL=2:CDLR=3:WR=12:nbkgrp=4:CCDL=2:RTPL=4" + +# Timing for 715 MHZ, Tesla Pascal P100 HBM runs at 715 MHZ +-gpgpu_dram_timing_opt "nbk=16:CCD=1:RRD=3:RCD=10:RAS=24:RP=10:RC=34: + CL=10:WL=2:CDLR=3:WR=9:nbkgrp=4:CCDL=2:RTPL=3" + +# Fermi has two schedulers per core +-gpgpu_num_sched_per_core 4 +# Two Level Scheduler with active and pending pools +#-gpgpu_scheduler two_level_active:6:0:1 +# Loose round robbin scheduler +#-gpgpu_scheduler lrr +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 +-visualizer_enabled 0 + +# power model configs +-power_simulation_enabled 0 +-gpuwattch_xml_file gpuwattch_gtx480.xml + +# tracing functionality +#-trace_enabled 1 +#-trace_components WARP_SCHEDULER,SCOREBOARD +#-trace_sampling_core 0 + diff --git a/configs/deprecated-cfgs/TeslaC2050/config_fermi_islip.icnt b/configs/deprecated-cfgs/TeslaC2050/config_fermi_islip.icnt new file mode 100644 index 0000000..a11bd8e --- /dev/null +++ b/configs/deprecated-cfgs/TeslaC2050/config_fermi_islip.icnt @@ -0,0 +1,70 @@ +//20*1 fly with 32 flits per packet under gpgpusim injection mode +use_map = 0; +flit_size = 32; + +// currently we donot use this, see subnets below +network_count = 2; + +// Topology +topology = fly; +k = 26; +n = 1; + +// Routing + +routing_function = dest_tag; + +// Flow control + +num_vcs = 1; +vc_buf_size = 8; + +wait_for_tail_credit = 0; + +// Router architecture + +vc_allocator = islip; //separable_input_first; +sw_allocator = islip; //separable_input_first; +alloc_iters = 1; + +credit_delay = 0; +routing_delay = 0; +vc_alloc_delay = 1; +sw_alloc_delay = 1; + +input_speedup = 2; +output_speedup = 1; +internal_speedup = 1.0; + +// Traffic, GPGPU-Sim does not use this + +traffic = uniform; +packet_size ={{1,2,3,4},{10,20}}; +packet_size_rate={{1,1,1,1},{2,1}}; + +// Simulation - Don't change + +sim_type = gpgpusim; +//sim_type = latency; +injection_rate = 0.1; + + +subnets = 2; + +// Always use read and write no matter following line +//use_read_write = 1; + + +read_request_subnet = 0; +read_reply_subnet = 1; +write_request_subnet = 0; +write_reply_subnet = 1; + +read_request_begin_vc = 0; +read_request_end_vc = 0; +write_request_begin_vc = 0; +write_request_end_vc = 0; +read_reply_begin_vc = 0; +read_reply_end_vc = 0; +write_reply_begin_vc = 0; +write_reply_end_vc = 0; diff --git a/configs/deprecated-cfgs/TeslaC2050/gpgpusim.config b/configs/deprecated-cfgs/TeslaC2050/gpgpusim.config new file mode 100644 index 0000000..6ac2c12 --- /dev/null +++ b/configs/deprecated-cfgs/TeslaC2050/gpgpusim.config @@ -0,0 +1,133 @@ +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 20 + +# Using cuobjdump to extract ptx/SASS +#-gpgpu_ptx_use_cuobjdump 1 # use default + +# SASS execution (only supported with CUDA >= 4.0) +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 + + +# high level architecture configuration +-gpgpu_n_clusters 14 +-gpgpu_n_cores_per_cluster 1 +-gpgpu_n_mem 6 +-gpgpu_n_sub_partition_per_mchannel 2 + +# Fermi clock domains +#-gpgpu_clock_domains ::: +# In Fermi, each pipeline has 16 execution units, so the Core clock needs to be divided +# by 2. (GPGPU-Sim simulates a warp (32 threads) in a single cycle). 1400/2 = 700 +-gpgpu_clock_domains 575.0:575.0:575.0:750.0 + +# shader core pipeline config +-gpgpu_shader_registers 32768 +-gpgpu_occupancy_sm_number 20 + +# This implies a maximum of 48 warps/SM +-gpgpu_shader_core_pipeline 1536:32 +-gpgpu_shader_cta 8 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB +-gpgpu_pipeline_widths 2,1,1,2,1,1,2 +-gpgpu_num_sp_units 2 +-gpgpu_num_sfu_units 1 + +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +-ptx_opcode_latency_int 4,13,4,5,145 +-ptx_opcode_initiation_int 1,2,2,1,8 +-ptx_opcode_latency_fp 4,13,4,5,39 +-ptx_opcode_initiation_fp 1,2,1,1,4 +-ptx_opcode_latency_dp 8,19,8,8,330 +-ptx_opcode_initiation_dp 8,16,8,8,130 + + +# In Fermi, the cache and shared memory can be configured to 16kb:48kb(default) or 48kb:16kb +# ::,::::,::,:** +# ** Optional parameter - Required when mshr_type==Texture Fifo +# Note: Hashing set index function (H) only applies to a set size of 32 or 64. +-gpgpu_cache:dl1 32:128:4,L:L:m:N:H,A:32:8,8 +-gpgpu_shmem_size 49152 + +# The alternative configuration for fermi in case cudaFuncCachePreferL1 is selected +#-gpgpu_cache:dl1 64:128:6,L:L:m:N:H,A:32:8,8 +#-gpgpu_shmem_size 16384 + +# 64 sets, each 128 bytes 8-way for each memory sub partition. This gives 786KB L2 cache +-gpgpu_cache:dl2 64:128:8,L:B:m:W:L,A:32:4,4:0,32 +-gpgpu_cache:dl2_texture_only 0 + +-gpgpu_cache:il1 4:128:4,L:R:f:N:L,A:2:32,4 +-gpgpu_tex_cache:l1 4:128:24,L:R:m:N:L,F:128:4,128:2 +-gpgpu_const_cache:l1 64:64:2,L:R:f:N:L,A:2:32,4 + +# enable operand collector +-gpgpu_operand_collector_num_units_sp 6 +-gpgpu_operand_collector_num_units_sfu 8 +-gpgpu_operand_collector_num_in_ports_sp 2 +-gpgpu_operand_collector_num_out_ports_sp 2 +-gpgpu_num_reg_banks 16 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 + +-gpgpu_max_insn_issue_per_warp 1 + +# interconnection +-network_mode 1 +-inter_config_file config_fermi_islip.icnt + +# memory partition latency config +-rop_latency 120 +-dram_latency 100 + +# dram model config +-gpgpu_dram_scheduler 1 +# The DRAM return queue and the scheduler queue together should provide buffer +# to sustain the memory level parallelism to tolerate DRAM latency +# To allow 100% DRAM utility, there should at least be enough buffer to sustain +# the minimum DRAM latency (100 core cycles). I.e. +# Total buffer space required = 100 x 750MHz / 575MHz = 130 +-gpgpu_frfcfs_dram_sched_queue_size 16 +-gpgpu_dram_return_queue_size 114 + +# for Fermi, bus width is 384bits, this is 8 bytes (4 bytes at each DRAM chip) per memory partition +-gpgpu_n_mem_per_ctrlr 2 +-gpgpu_dram_buswidth 4 +-gpgpu_dram_burst_length 8 +-dram_data_command_freq_ratio 4 # GDDR5 is QDR +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.BBBCCCCB.CCSSSSSS + +# GDDR5 timing from hynix H5GQ1H24AFR +# to disable bank groups, set nbkgrp to 1 and tCCDL and tRTPL to 0 +-gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=6:RCD=12:RAS=28:RP=12:RC=40: + CL=12:WL=4:CDLR=5:WR=12:nbkgrp=4:CCDL=3:RTPL=2" + +# Fermi has two schedulers per core +-gpgpu_num_sched_per_core 2 +# Two Level Scheduler with active and pending pools +#-gpgpu_scheduler two_level_active:6:0:1 +# Loose round robbin scheduler +#-gpgpu_scheduler lrr +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 +-visualizer_enabled 0 + +# tracing functionality +#-trace_enabled 1 +#-trace_components WARP_SCHEDULER,SCOREBOARD +#-trace_sampling_core 0 diff --git a/configs/tested-cfgs/SM2_GTX480/config_fermi_islip.icnt b/configs/tested-cfgs/SM2_GTX480/config_fermi_islip.icnt new file mode 100644 index 0000000..c399db9 --- /dev/null +++ b/configs/tested-cfgs/SM2_GTX480/config_fermi_islip.icnt @@ -0,0 +1,70 @@ +//21*1 fly with 32 flits per packet under gpgpusim injection mode +use_map = 0; +flit_size = 40; + +// currently we do not use this, see subnets below +network_count = 2; + +// Topology +topology = fly; +k = 27; +n = 1; + +// Routing + +routing_function = dest_tag; + +// Flow control + +num_vcs = 1; +vc_buf_size = 64; + +wait_for_tail_credit = 0; + +// Router architecture + +vc_allocator = islip; //separable_input_first; +sw_allocator = islip; //separable_input_first; +alloc_iters = 1; + +credit_delay = 0; +routing_delay = 0; +vc_alloc_delay = 1; +sw_alloc_delay = 1; + +input_speedup = 1; +output_speedup = 1; +internal_speedup = 2.0; + +// Traffic, GPGPU-Sim does not use this + +traffic = uniform; +packet_size ={{1,2,3,4},{10,20}}; +packet_size_rate={{1,1,1,1},{2,1}}; + +// Simulation - Don't change + +sim_type = gpgpusim; +//sim_type = latency; +injection_rate = 0.1; + +subnets = 2; + +// Always use read and write no matter following line +//use_read_write = 1; + + +read_request_subnet = 0; +read_reply_subnet = 1; +write_request_subnet = 0; +write_reply_subnet = 1; + +read_request_begin_vc = 0; +read_request_end_vc = 0; +write_request_begin_vc = 0; +write_request_end_vc = 0; +read_reply_begin_vc = 0; +read_reply_end_vc = 0; +write_reply_begin_vc = 0; +write_reply_end_vc = 0; + diff --git a/configs/tested-cfgs/SM2_GTX480/gpgpusim.config b/configs/tested-cfgs/SM2_GTX480/gpgpusim.config new file mode 100644 index 0000000..4096b09 --- /dev/null +++ b/configs/tested-cfgs/SM2_GTX480/gpgpusim.config @@ -0,0 +1,153 @@ +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 20 + + +# SASS execution (only supported with CUDA >= 4.0) +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 + +# high level architecture configuration +-gpgpu_n_clusters 15 +-gpgpu_n_cores_per_cluster 1 +-gpgpu_n_mem 6 +-gpgpu_n_sub_partition_per_mchannel 2 + +# Fermi clock domains +#-gpgpu_clock_domains ::: +# In Fermi, each pipeline has 16 execution units, so the Core clock needs to be divided +# by 2. (GPGPU-Sim simulates a warp (32 threads) in a single cycle). 1400/2 = 700 +-gpgpu_clock_domains 700.0:700.0:700.0:924.0 + +# shader core pipeline config +-gpgpu_shader_registers 32768 +-gpgpu_occupancy_sm_number 20 + +# This implies a maximum of 48 warps/SM +-gpgpu_shader_core_pipeline 1536:32 +-gpgpu_shader_cta 8 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB +#For Fermi, DP unit =0, DP inst is executed on SFU +-gpgpu_pipeline_widths 2,0,1,1,2,0,1,1,2 +-gpgpu_num_sp_units 2 +-gpgpu_num_sfu_units 1 +-gpgpu_num_dp_units 0 + +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +-ptx_opcode_latency_int 4,13,4,5,145 +-ptx_opcode_initiation_int 1,2,2,1,8 +-ptx_opcode_latency_fp 4,13,4,5,39 +-ptx_opcode_initiation_fp 1,2,1,1,4 +-ptx_opcode_latency_dp 8,19,8,8,330 +-ptx_opcode_initiation_dp 8,16,8,8,130 + + +# In Fermi, the cache and shared memory can be configured to 16kb:48kb(default) or 48kb:16kb +# ::,::::,::,:** +# ** Optional parameter - Required when mshr_type==Texture Fifo +# Note: Hashing set index function (H) only applies to a set size of 32 or 64. +-gpgpu_cache:dl1 N:32:128:4,L:L:m:N:H,S:64:8,8 +-gpgpu_shmem_size 49152 +-icnt_flit_size 40 +-gmem_skip_L1D 0 +-gpgpu_n_cluster_ejection_buffer_size 32 +-l1_latency 35 +-smem_latency 26 +-gpgpu_flush_l1_cache 1 + +# The alternative configuration for fermi in case cudaFuncCachePreferL1 is selected +#-gpgpu_cache:dl1 N:64:128:6,L:L:m:N:H,S:64:8,8 +#-gpgpu_shmem_size 16384 + +# 64 sets, each 128 bytes 8-way for each memory sub partition. This gives 786KB L2 cache +-gpgpu_cache:dl2 S:64:128:8,L:B:m:L:L,A:256:4,4:0,32 +-gpgpu_cache:dl2_texture_only 0 +-gpgpu_dram_partition_queues 64:64:64:64 +-perf_sim_memcpy 1 +-memory_partition_indexing 0 + +-gpgpu_cache:il1 N:4:128:4,L:R:f:N:L,S:2:32,4 +-gpgpu_tex_cache:l1 N:4:128:24,L:R:m:N:L,T:128:4,128:2 +-gpgpu_const_cache:l1 N:64:64:2,L:R:f:N:L,S:2:32,4 + +# enable operand collector +-gpgpu_operand_collector_num_units_sp 6 +-gpgpu_operand_collector_num_units_sfu 8 +-gpgpu_operand_collector_num_in_ports_sp 2 +-gpgpu_operand_collector_num_out_ports_sp 2 +-gpgpu_num_reg_banks 16 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 +-gpgpu_coalesce_arch 20 + +-gpgpu_max_insn_issue_per_warp 1 + +# interconnection +-network_mode 1 +-inter_config_file config_fermi_islip.icnt + +# memory partition latency config +-rop_latency 120 +-dram_latency 100 + +# dram model config +-gpgpu_dram_scheduler 1 +# The DRAM return queue and the scheduler queue together should provide buffer +# to sustain the memory level parallelism to tolerate DRAM latency +# To allow 100% DRAM utility, there should at least be enough buffer to sustain +# the minimum DRAM latency (100 core cycles). I.e. +# Total buffer space required = 100 x 924MHz / 700MHz = 132 +-gpgpu_frfcfs_dram_sched_queue_size 64 +-gpgpu_dram_return_queue_size 116 + +# for Fermi, bus width is 384bits, this is 8 bytes (4 bytes at each DRAM chip) per memory partition +-gpgpu_n_mem_per_ctrlr 2 +-gpgpu_dram_buswidth 4 +-gpgpu_dram_burst_length 8 +-dram_data_command_freq_ratio 4 # GDDR5 is QDR +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.BBBCCCCB.CCSSSSSS + +# GDDR5 timing from hynix H5GQ1H24AFR +# to disable bank groups, set nbkgrp to 1 and tCCDL and tRTPL to 0 +-gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=6:RCD=12:RAS=28:RP=12:RC=40: + CL=12:WL=4:CDLR=5:WR=12:nbkgrp=4:CCDL=3:RTPL=2" + +# select lower bits for bnkgrp to increase bnkgrp parallelism +-dram_bnk_indexing_policy 0 +-dram_bnkgrp_indexing_policy 1 + +#-Seperate_Write_Queue_Enable 1 +#-Write_Queue_Size 64:56:32 + +# Fermi has two schedulers per core +-gpgpu_num_sched_per_core 2 +# Two Level Scheduler with active and pending pools +#-gpgpu_scheduler two_level_active:6:0:1 +# Loose round robbin scheduler +#-gpgpu_scheduler lrr +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 +-visualizer_enabled 0 + +# power model configs +-power_simulation_enabled 1 +-gpuwattch_xml_file gpuwattch_gtx480.xml + +# tracing functionality +#-trace_enabled 1 +#-trace_components WARP_SCHEDULER,SCOREBOARD +#-trace_sampling_core 0 diff --git a/configs/tested-cfgs/SM2_GTX480/gpuwattch_gtx480.xml b/configs/tested-cfgs/SM2_GTX480/gpuwattch_gtx480.xml new file mode 100755 index 0000000..304e0fd --- /dev/null +++ b/configs/tested-cfgs/SM2_GTX480/gpuwattch_gtx480.xml @@ -0,0 +1,538 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/configs/tested-cfgs/SM6_TITANX/config_fermi_islip.icnt b/configs/tested-cfgs/SM6_TITANX/config_fermi_islip.icnt new file mode 100644 index 0000000..dec4789 --- /dev/null +++ b/configs/tested-cfgs/SM6_TITANX/config_fermi_islip.icnt @@ -0,0 +1,73 @@ +//21*1 fly with 32 flits per packet under gpgpusim injection mode +use_map = 0; +flit_size = 40; + +// currently we do not use this, see subnets below +network_count = 2; + +// Topology +topology = fly; +k = 52; +n = 1; + +// Routing + +routing_function = dest_tag; + +// Flow control + +num_vcs = 1; +vc_buf_size = 64; +input_buffer_size = 256; +ejection_buffer_size = 64; +boundary_buffer_size = 64; + +wait_for_tail_credit = 0; + +// Router architecture + +vc_allocator = islip; //separable_input_first; +sw_allocator = islip; //separable_input_first; +alloc_iters = 1; + +credit_delay = 0; +routing_delay = 0; +vc_alloc_delay = 1; +sw_alloc_delay = 1; + +input_speedup = 1; +output_speedup = 1; +internal_speedup = 2.0; + +// Traffic, GPGPU-Sim does not use this + +traffic = uniform; +packet_size ={{1,2,3,4},{10,20}}; +packet_size_rate={{1,1,1,1},{2,1}}; + +// Simulation - Don't change + +sim_type = gpgpusim; +//sim_type = latency; +injection_rate = 0.1; + +subnets = 2; + +// Always use read and write no matter following line +//use_read_write = 1; + + +read_request_subnet = 0; +read_reply_subnet = 1; +write_request_subnet = 0; +write_reply_subnet = 1; + +read_request_begin_vc = 0; +read_request_end_vc = 0; +write_request_begin_vc = 0; +write_request_end_vc = 0; +read_reply_begin_vc = 0; +read_reply_end_vc = 0; +write_reply_begin_vc = 0; +write_reply_end_vc = 0; + diff --git a/configs/tested-cfgs/SM6_TITANX/gpgpusim.config b/configs/tested-cfgs/SM6_TITANX/gpgpusim.config new file mode 100644 index 0000000..3842508 --- /dev/null +++ b/configs/tested-cfgs/SM6_TITANX/gpgpusim.config @@ -0,0 +1,185 @@ +# This config models the Pascal GP102 (NVIDIA TITAN X) +# For more info about this card, see Nvidia White paper +# http://international.download.nvidia.com/geforce-com/international/pdfs/GeForce_GTX_1080_Whitepaper_FINAL.pdf + +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 61 +-gpgpu_ignore_resources_limitation 1 + +# SASS execution (only supported with CUDA >= 4.0) +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 + +# high level architecture configuration +# P102 has two semi-indp scheds per core, and two cores per cluster +-gpgpu_n_clusters 28 +-gpgpu_n_cores_per_cluster 2 +-gpgpu_n_mem 12 +-gpgpu_n_sub_partition_per_mchannel 2 + +# Pascal clock domains +#-gpgpu_clock_domains ::: +# Pascal NVIDIA TITAN X clock domains are adopted from +# https://en.wikipedia.org/wiki/GeForce_10_series +-gpgpu_clock_domains 1417.0:1417.0:1417.0:2500.0 + +# shader core pipeline config +-gpgpu_shader_registers 32768 +-gpgpu_occupancy_sm_number 62 + +# This implies a maximum of 32 warps/SM +-gpgpu_shader_core_pipeline 1024:32 +-gpgpu_shader_cta 16 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB +## Pascal GP102 has 4 SP SIMD units and 4 SFU units +## we need to scale the number of pipeline registers to be equal to the number of SP units +-gpgpu_pipeline_widths 2,1,2,1,2,1,2,1,5 +-gpgpu_num_sp_units 2 +-gpgpu_num_sfu_units 2 +-gpgpu_num_dp_units 1 + + +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +# All Div operations are executed on SFU unit +# Throughput (initiation latency) are adopted from CUDA SDK document V8, section 5.4.1, Table 2 +-ptx_opcode_latency_int 4,13,4,5,145 +-ptx_opcode_initiation_int 1,1,1,1,4 +-ptx_opcode_latency_fp 4,13,4,5,39 +-ptx_opcode_initiation_fp 1,2,1,1,4 +-ptx_opcode_latency_dp 8,19,8,8,330 +-ptx_opcode_initiation_dp 8,8,8,8,130 +-ptx_opcode_initiation_sfu 4 +-ptx_opcode_latency_sfu 8 + + +# latencies and cache configs are adopted from: +# https://arxiv.org/pdf/1804.06826.pdf +# ::,::::,::,:** +# ** Optional parameter - Required when mshr_type==Texture Fifo +# Note: Hashing set index function (H) only applies to a set size of 32 or 64. +# Pascal GP102 has 96KB Shared memory divided over 2 cores, each has 48KB +# Pascal GP102 has 2 banks L1 cache, where each is 24KB L1 cache +# The defulat is to disable the L1 cache, unless cache modifieres are used +-gpgpu_cache:dl1 S:4:128:48,L:L:s:N:L,A:256:8,16:0,32 +-gpgpu_cache:dl1PrefL1 S:4:128:48,L:L:s:N:L,A:256:8,16:0,32 +-gpgpu_cache:dl1PrefShared S:4:128:48,L:L:s:N:L,A:256:8,16:0,32 +-gpgpu_shmem_size 49152 +-gpgpu_shmem_size_PrefL1 49152 +-gpgpu_shmem_size_PrefShared 49152 +# By default, L1 cache is disabled in Pascal P102. +# requests with .nc modifier or __ldg mehtod will be cached in L1 cache even with gmem_skip_L1D=1 +-gmem_skip_L1D 1 +-icnt_flit_size 40 +-gpgpu_n_cluster_ejection_buffer_size 32 +-l1_latency 82 +-smem_latency 24 +-gpgpu_flush_l1_cache 1 + +# 64 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 3MB L2 cache +-gpgpu_cache:dl2 S:64:128:16,L:B:m:L:L,A:256:64,16:0,32 +-gpgpu_cache:dl2_texture_only 0 +-gpgpu_dram_partition_queues 32:32:32:32 +-perf_sim_memcpy 1 + +# 4 KB Inst. +-gpgpu_cache:il1 N:8:128:4,L:R:f:N:L,S:2:48,4 +# 48 KB Tex +# Note, TEX is deprected in Volta, It is used for legacy apps only. Use L1D cache instead with .nc modifier or __ldg mehtod +-gpgpu_tex_cache:l1 N:16:128:24,L:R:m:N:L,T:128:4,128:2 +# 12 KB Const +-gpgpu_const_cache:l1 N:128:64:2,L:R:f:N:L,S:2:64,4 + +# enable operand collector +-gpgpu_operand_collector_num_units_sp 12 +-gpgpu_operand_collector_num_units_sfu 6 +-gpgpu_operand_collector_num_units_mem 8 +-gpgpu_operand_collector_num_units_dp 6 +-gpgpu_operand_collector_num_in_ports_sp 2 +-gpgpu_operand_collector_num_out_ports_sp 2 +-gpgpu_operand_collector_num_in_ports_sfu 2 +-gpgpu_operand_collector_num_out_ports_sfu 2 +-gpgpu_operand_collector_num_in_ports_mem 1 +-gpgpu_operand_collector_num_out_ports_mem 1 +-gpgpu_operand_collector_num_in_ports_dp 1 +-gpgpu_operand_collector_num_out_ports_dp 1 +-gpgpu_num_reg_banks 32 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 +# Use Pascal Coalsce arhitetecture +-gpgpu_coalesce_arch 61 + +## In Pascal, a warp scheduler can issue 2 insts per cycle using 2 diff execution units +-gpgpu_max_insn_issue_per_warp 2 +-gpgpu_dual_issue_diff_exec_units 1 + +# interconnection +-network_mode 1 +-inter_config_file config_fermi_islip.icnt + +# memory partition latency config +-rop_latency 120 +-dram_latency 100 + +# dram model config +-gpgpu_dram_scheduler 1 +# The DRAM return queue and the scheduler queue together should provide buffer +# to sustain the memory level parallelism to tolerate DRAM latency +# To allow 100% DRAM utility, there should at least be enough buffer to sustain +# the minimum DRAM latency (100 core cycles). I.e. +# Total buffer space required = 100 x 924MHz / 700MHz = 132 +-gpgpu_frfcfs_dram_sched_queue_size 64 +-gpgpu_dram_return_queue_size 240 + +# for NVIDIA TITAN X, bus width is 384bits (12 DRAM chips x 32 bits) +# 12 memory paritions, 4 bytes (1 DRAM chip) per memory partition +# the atom size of GDDR5X (the smallest read request) is 32 bytes +-gpgpu_n_mem_per_ctrlr 1 +-gpgpu_dram_buswidth 4 +-gpgpu_dram_burst_length 8 +-dram_data_command_freq_ratio 4 # GDDR5X is QDR +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS + +# Use the same GDDR5 timing +-gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=8:RCD=16:RAS=37:RP=16:RC=52: + CL=16:WL=6:CDLR=7:WR=16:nbkgrp=4:CCDL=4:RTPL=3" + +-dram_bnk_indexing_policy 0 +-dram_bnkgrp_indexing_policy 1 + +#-Seperate_Write_Queue_Enable 1 +#-Write_Queue_Size 64:56:32 + +# Pascal 102 has four schedulers per core +-gpgpu_num_sched_per_core 2 +# Two Level Scheduler with active and pending pools +#-gpgpu_scheduler two_level_active:6:0:1 +# Loose round robbin scheduler +#-gpgpu_scheduler lrr +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 +-visualizer_enabled 0 + +# power model configs, disable it untill we create a real energy model for Pascal 102 +-power_simulation_enabled 0 +-gpuwattch_xml_file gpuwattch_gtx480.xml + +# tracing functionality +#-trace_enabled 1 +#-trace_components WARP_SCHEDULER,SCOREBOARD +#-trace_sampling_core 0 + diff --git a/configs/tested-cfgs/SM7_TITANV/config_fermi_islip.icnt b/configs/tested-cfgs/SM7_TITANV/config_fermi_islip.icnt new file mode 100644 index 0000000..2f25889 --- /dev/null +++ b/configs/tested-cfgs/SM7_TITANV/config_fermi_islip.icnt @@ -0,0 +1,74 @@ +//21*1 fly with 32 flits per packet under gpgpusim injection mode +use_map = 0; +flit_size = 40; + +// currently we do not use this, see subnets below +network_count = 2; + +// Topology +topology = fly; +k = 64; +n = 1; + +// Routing + +routing_function = dest_tag; + + +// Flow control + +num_vcs = 1; +vc_buf_size = 256; +input_buffer_size = 256; +ejection_buffer_size = 256; +boundary_buffer_size = 256; + +wait_for_tail_credit = 0; + +// Router architecture + +vc_allocator = islip; //separable_input_first; +sw_allocator = islip; //separable_input_first; +alloc_iters = 1; + +credit_delay = 0; +routing_delay = 0; +vc_alloc_delay = 1; +sw_alloc_delay = 1; + +input_speedup = 1; +output_speedup = 1; +internal_speedup = 2.0; + +// Traffic, GPGPU-Sim does not use this + +traffic = uniform; +packet_size ={{1,2,3,4},{10,20}}; +packet_size_rate={{1,1,1,1},{2,1}}; + +// Simulation - Don't change + +sim_type = gpgpusim; +//sim_type = latency; +injection_rate = 0.1; + +subnets = 2; + +// Always use read and write no matter following line +//use_read_write = 1; + + +read_request_subnet = 0; +read_reply_subnet = 1; +write_request_subnet = 0; +write_reply_subnet = 1; + +read_request_begin_vc = 0; +read_request_end_vc = 0; +write_request_begin_vc = 0; +write_request_end_vc = 0; +read_reply_begin_vc = 0; +read_reply_end_vc = 0; +write_reply_begin_vc = 0; +write_reply_end_vc = 0; + diff --git a/configs/tested-cfgs/SM7_TITANV/gpgpusim.config b/configs/tested-cfgs/SM7_TITANV/gpgpusim.config new file mode 100644 index 0000000..c8351da --- /dev/null +++ b/configs/tested-cfgs/SM7_TITANV/gpgpusim.config @@ -0,0 +1,191 @@ +# This config models the Volta Titan X +# For more info about this card: +# http://images.nvidia.com/content/volta-architecture/pdf/volta-architecture-whitepaper.pdf +# https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=8344474&tag=1# +# http://on-demand.gputechconf.com/gtc/2018/presentation/s8122-dissecting-the-volta-gpu-architecture-through-microbenchmarking.pdf +# https://en.wikipedia.org/wiki/Volta_(microarchitecture) +# https://www.hotchips.org/wp-content/uploads/hc_archives/hc29/HC29.21-Monday-Pub/HC29.21.10-GPU-Gaming-Pub/HC29.21.132-Volta-Choquette-NVIDIA-Final3.pdf +# https://devblogs.nvidia.com/inside-volta/ +# http://on-demand.gputechconf.com/gtc/2017/presentation/s7798-luke-durant-inside-volta.pdf + +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 70 + +# SASS execution (only supported with CUDA >= 4.0) +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 + +# high level architecture configuration +-gpgpu_n_clusters 40 +-gpgpu_n_cores_per_cluster 2 +-gpgpu_n_mem 24 +-gpgpu_n_sub_partition_per_mchannel 1 + +# volta clock domains +#-gpgpu_clock_domains ::: +# Volta NVIDIA GV100 clock domains are adopted from +# https://en.wikipedia.org/wiki/Volta_(microarchitecture) +-gpgpu_clock_domains 1200.0:2000.0:1200.0:850.0 +# boost mode +# -gpgpu_clock_domains 1455.0:2000.0:1455.0:850.0 + +# shader core pipeline config +-gpgpu_shader_registers 65536 +-gpgpu_occupancy_sm_number 70 + +# This implies a maximum of 64 warps/SM +-gpgpu_shader_core_pipeline 2048:32 +-gpgpu_shader_cta 32 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB +## Volta GV100 has 2 SP SIMD units, 2 SFU units, 2 DP units per core +## we need to scale the number of pipeline registers to be equal to the number of SP units +-gpgpu_pipeline_widths 4,4,4,1,4,4,4,1,12 +-gpgpu_num_sp_units 4 +-gpgpu_num_sfu_units 4 +-gpgpu_num_dp_units 4 + +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +# All Div operations are executed on SFU unit +# Throughput (initiation latency) are adopted from +# http://on-demand.gputechconf.com/gtc/2018/presentation/s8122-dissecting-the-volta-gpu-architecture-through-microbenchmarking.pdf +-ptx_opcode_latency_int 4,13,4,5,145 +-ptx_opcode_initiation_int 2,2,2,2,8 +-ptx_opcode_latency_fp 4,13,4,5,39 +-ptx_opcode_initiation_fp 2,2,2,2,4 +-ptx_opcode_latency_dp 8,19,8,8,330 +-ptx_opcode_initiation_dp 4,4,4,4,130 +-ptx_opcode_latency_sfu 100 +-ptx_opcode_initiation_sfu 8 + + +# ::,::::,::,:** +# ** Optional parameter - Required when mshr_type==Texture Fifo +# Defualt config is 32KB DL1 and 96KB shared memory +# In Volta, we assign the remaining shared memory to L1 cache +# if the assigned shd mem = 0, then L1 cache = 128KB +# For more info, see https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#shared-memory-7-x +# disable this mode in case of multi kernels/apps execution +-adpative_volta_cache_config 1 +-gpgpu_cache:dl1 S:4:128:64,L:L:s:N:L,A:256:8,16:0,32 +-gpgpu_shmem_size 98304 +-gmem_skip_L1D 0 +-icnt_flit_size 40 +-gpgpu_n_cluster_ejection_buffer_size 32 +-l1_latency 28 +-smem_latency 19 +-gpgpu_flush_l1_cache 1 + +# 64 sets, each 128 bytes 24-way for each memory sub partition (192 KB per memory sub partition). This gives 4.5MB L2 cache +-gpgpu_cache:dl2 S:64:128:24,L:B:m:L:L,A:384:4,32:0,32 +-gpgpu_cache:dl2_texture_only 0 +-gpgpu_dram_partition_queues 64:64:64:64 +-perf_sim_memcpy 1 + +# 128 KB Inst. +-gpgpu_cache:il1 N:64:128:16,L:R:f:N:L,S:2:48,4 +# 48 KB Tex +# Note, TEX is deprected in Volta, It is used for legacy apps only. Use L1D cache instead with .nc modifier or __ldg mehtod +-gpgpu_tex_cache:l1 N:16:128:24,L:R:m:N:L,T:128:4,128:2 +# 64 KB Const +-gpgpu_const_cache:l1 N:128:64:8,L:R:f:N:L,S:2:64,4 + +# enable operand collector +-gpgpu_operand_collector_num_units_sp 12 +-gpgpu_operand_collector_num_units_sfu 6 +-gpgpu_operand_collector_num_units_mem 8 +-gpgpu_operand_collector_num_units_dp 6 +-gpgpu_operand_collector_num_in_ports_sp 4 +-gpgpu_operand_collector_num_out_ports_sp 4 +-gpgpu_operand_collector_num_in_ports_sfu 1 +-gpgpu_operand_collector_num_out_ports_sfu 1 +-gpgpu_operand_collector_num_in_ports_mem 1 +-gpgpu_operand_collector_num_out_ports_mem 1 +-gpgpu_operand_collector_num_in_ports_dp 1 +-gpgpu_operand_collector_num_out_ports_dp 1 +-gpgpu_num_reg_banks 32 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 +-gpgpu_coalesce_arch 60 + +## In Pascal, a warp scheduler can issue 2 insts per cycle using 2 diff execution units +-gpgpu_max_insn_issue_per_warp 1 +-gpgpu_dual_issue_diff_exec_units 1 + +# interconnection +-network_mode 1 +-inter_config_file config_fermi_islip.icnt + +# memory partition latency config +-rop_latency 120 +-dram_latency 100 + +# dram model config +-gpgpu_dram_scheduler 1 +# The DRAM return queue and the scheduler queue together should provide buffer +# to sustain the memory level parallelism to tolerate DRAM latency +# To allow 100% DRAM utility, there should at least be enough buffer to sustain +# the minimum DRAM latency (100 core cycles). I.e. +# Total buffer space required = 100 x 924MHz / 700MHz = 132 +-gpgpu_frfcfs_dram_sched_queue_size 64 +-gpgpu_dram_return_queue_size 192 + +# for HBM, 32 channles, each (128 bits) 16 bytes width +-gpgpu_n_mem_per_ctrlr 1 +-gpgpu_dram_buswidth 16 +-gpgpu_dram_burst_length 2 +-dram_data_command_freq_ratio 2 # HBM is DDR +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBBCCC.CCCSSSSS + +# HBM timing are adopted from hynix JESD235 standered and nVidia HPCA 2017 paper (http://www.cs.utah.edu/~nil/pubs/hpca17.pdf) +# Timing for 1 GHZ +# tRRDl and tWTR are missing, need to be added +#-gpgpu_dram_timing_opt "nbk=16:CCD=1:RRD=4:RCD=14:RAS=33:RP=14:RC=47: +# CL=14:WL=2:CDLR=3:WR=12:nbkgrp=4:CCDL=2:RTPL=4" + +# Timing for 715 MHZ, Tesla Volta V100 HBM runs at 715 MHZ +-gpgpu_dram_timing_opt "nbk=16:CCD=1:RRD=3:RCD=10:RAS=24:RP=10:RC=34: + CL=10:WL=2:CDLR=3:WR=9:nbkgrp=4:CCDL=2:RTPL=3" + +# HBM has dual bus interface, in which it can issue two col and row commands at a time +-dual_bus_interface 1 +# select lower bits for bnkgrp to increase bnkgrp parallelism +-dram_bnk_indexing_policy 0 +-dram_bnkgrp_indexing_policy 1 + +#-Seperate_Write_Queue_Enable 1 +#-Write_Queue_Size 64:56:32 + +# Pascal has two schedulers per core +-gpgpu_num_sched_per_core 4 +# Two Level Scheduler with active and pending pools +#-gpgpu_scheduler two_level_active:6:0:1 +# Loose round robbin scheduler +#-gpgpu_scheduler lrr +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 +-visualizer_enabled 0 + +# power model configs, disable it untill we create a real energy model for Pascal 100 +-power_simulation_enabled 0 +-gpuwattch_xml_file gpuwattch_gtx480.xml + +# tracing functionality +#-trace_enabled 1 +#-trace_components WARP_SCHEDULER,SCOREBOARD +#-trace_sampling_core 0 + diff --git a/cuobjdump_to_ptxplus/ptx_parser.h b/cuobjdump_to_ptxplus/ptx_parser.h index a534e92..ee7a942 100644 --- a/cuobjdump_to_ptxplus/ptx_parser.h +++ b/cuobjdump_to_ptxplus/ptx_parser.h @@ -376,4 +376,8 @@ void func_header_info_int(const char* s, int i) g_headerList->getListEnd().addOperand(buff); } } + +void maxnt_id(int x, int y, int z) { + +} #endif //_PTX_PARSER_H_ diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index d67fd85..61af0ee 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -338,10 +338,19 @@ struct _cuda_device_id *GPGPUSim_Init() prop->minor = 2; prop->totalGlobalMem = 0x80000000 /* 2 GB */; prop->memPitch = 0; - prop->maxThreadsPerBlock = 512; - prop->maxThreadsDim[0] = 512; - prop->maxThreadsDim[1] = 512; - prop->maxThreadsDim[2] = 512; + if(prop->major >= 2) { + prop->maxThreadsPerBlock = 1024; + prop->maxThreadsDim[0] = 1024; + prop->maxThreadsDim[1] = 1024; + } + else + { + prop->maxThreadsPerBlock = 512; + prop->maxThreadsDim[0] = 512; + prop->maxThreadsDim[1] = 512; + } + + prop->maxThreadsDim[2] = 64; prop->maxGridSize[0] = 0x40000000; prop->maxGridSize[1] = 0x40000000; prop->maxGridSize[2] = 0x40000000; @@ -353,6 +362,9 @@ struct _cuda_device_id *GPGPUSim_Init() prop->clockRate = the_gpu->shader_clock(); #if (CUDART_VERSION >= 2010) prop->multiProcessorCount = the_gpu->get_config().num_shader(); +#endif +#if (CUDART_VERSION >= 4000) + prop->maxThreadsPerMultiProcessor = the_gpu->threads_per_core(); #endif the_gpu->set_prop(prop); the_device = new _cuda_device_id(the_gpu); @@ -2167,8 +2179,28 @@ cudaError_t CUDARTAPI cudaSetValidDevices(int *device_arr, int len) cudaError_t CUDARTAPI cudaSetDeviceFlags( int flags ) { - cuda_not_implemented(__my_func__,__LINE__); - return g_last_cudaError = cudaErrorUnknown; + // This flag is implicitly always on (unless you are using the driver API). It is safe for GPGPU-Sim to + // just ignore it. + if ( cudaDeviceMapHost == flags ) { + return g_last_cudaError = cudaSuccess; + } else { + cuda_not_implemented(__my_func__,__LINE__); + return g_last_cudaError = cudaErrorUnknown; + } +} + +size_t getMaxThreadsPerBlock(struct cudaFuncAttributes *attr) { + _cuda_device_id *dev = GPGPUSim_Init(); + struct cudaDeviceProp prop; + + prop = *dev->get_prop(); + + size_t max = prop.maxThreadsPerBlock; + + if ((prop.regsPerBlock / attr->numRegs) < max) + max = prop.regsPerBlock / attr->numRegs; + + return max; } cudaError_t CUDARTAPI cudaFuncGetAttributes(struct cudaFuncAttributes *attr, const char *hostFun ) @@ -2181,7 +2213,10 @@ cudaError_t CUDARTAPI cudaFuncGetAttributes(struct cudaFuncAttributes *attr, con attr->constSizeBytes = kinfo->cmem; attr->localSizeBytes = kinfo->lmem; attr->numRegs = kinfo->regs; - attr->maxThreadsPerBlock = 0; // from pragmas? + if(kinfo->maxthreads > 0) + attr->maxThreadsPerBlock = kinfo->maxthreads; + else + attr->maxThreadsPerBlock = getMaxThreadsPerBlock(attr); #if CUDART_VERSION >= 3000 attr->ptxVersion = kinfo->ptx_version; attr->binaryVersion = kinfo->sm_target; diff --git a/nightly.jenkinsfile b/nightly.jenkinsfile new file mode 100644 index 0000000..5221b3b --- /dev/null +++ b/nightly.jenkinsfile @@ -0,0 +1,75 @@ +pipeline { + agent { + label "purdue-cluster" + } + + options { + disableConcurrentBuilds() + overrideIndexTriggers(true) + } + + triggers { + pollSCM('0 1 * * *') + } + + stages { + stage('nightly-simulator-build') { + steps { + sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/9.1_env_setup.sh &&\ + source `pwd`/setup_environment &&\ + make -j' + } + } + stage('nightly-simulations-build'){ + steps{ + sh 'rm -rf gpgpu-sim_simulations' + sh 'git clone git@github.rcac.purdue.edu:TimRogersGroup/gpgpu-sim_simulations.git && \ + cd gpgpu-sim_simulations && \ + git checkout purdue-cluster && \ + git pull && \ + ln -s /home/tgrogers-raid/a/common/data_dirs benchmarks/' + sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/9.1_env_setup.sh &&\ + source `pwd`/setup_environment &&\ + cd gpgpu-sim_simulations && \ + source ./benchmarks/src/setup_environment && \ + make -i -j -C ./benchmarks/src/ all && \ + make -C ./benchmarks/src data' + } + } + stage('nightly-2B-insn-run'){ + steps { + sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/9.1_env_setup.sh &&\ + source `pwd`/setup_environment &&\ + ./gpgpu-sim_simulations/util/job_launching/run_simulations.py -B `cat ./gpgpu-sim_simulations/util/job_launching/apps/all-apps.list` -C TITANX-2B,TITANX-L1ON-2B,P100-2B,TITANV-2B -N nightly-$$ && \ + PLOTDIR="jenkins/${JOB_NAME}/${BUILD_NUMBER}/getstats" && ssh tgrogers@dynamo.ecn.purdue.edu mkdir -p /home/dynamo/a/tgrogers/website/gpgpu-sim-plots/$PLOTDIR && \ + ./gpgpu-sim_simulations/util/job_launching/monitor_func_test.py -I -S 1800 -v -s stats-$$.csv -N nightly-$$ && \ + ./gpgpu-sim_simulations/util/plotting/plot-get-stats.py -c stats-$$.csv -p tgrogers@dynamo.ecn.purdue.edu:~/website/gpgpu-sim-plots/$PLOTDIR -w https://engineering.purdue.edu/tgrogers/gpgpu-sim-plots/$PLOTDIR -n $PLOTDIR' + } + } + stage('nightly-correlate'){ + steps { + sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/9.1_env_setup.sh &&\ + source `pwd`/setup_environment &&\ + PLOTDIR="jenkins/${JOB_NAME}" &&\ + ./gpgpu-sim_simulations/util/job_launching/get_stats.py -R -K -k -B `cat ./gpgpu-sim_simulations/util/job_launching/apps/all-apps.list` -C TITANX-2B,TITANX-L1ON-2B,P100-2B,TITANV-2B > nightly-stats-per-kernel-9.1.csv &&\ + ./gpgpu-sim_simulations/util/plotting/correlate_and_publish.sh nightly-stats-per-kernel-9.1.csv $PLOTDIR ${BUILD_NUMBER}' + } + } + } + post { + success { + emailext body: "See ${BUILD_URL}", + recipientProviders: [[$class: 'CulpritsRecipientProvider'], + [$class: 'RequesterRecipientProvider']], + subject: "[AALP Jenkins] Build #${BUILD_NUMBER} - Success!", + to: 'tgrogers@purdue.edu' + } + failure { + emailext body: "See ${BUILD_URL}", + recipientProviders: [[$class: 'CulpritsRecipientProvider'], + [$class: 'RequesterRecipientProvider']], + subject: "[AALP Jenkins] Build #${BUILD_NUMBER} - ${currentBuild.result}", + to: 'tgrogers@purdue.edu' + } + } +} diff --git a/setup_environment b/setup_environment index f1af978..9578942 100644 --- a/setup_environment +++ b/setup_environment @@ -1,6 +1,6 @@ # see README before running this -ps -p $$ | awk '/bash/ || / sh/ || /zsh/ {exit 1;}' && echo "ERROR ** source setup_environment must be run in a bash, zsh or sh shell; see README" && exit +ps -p $$ | awk '/bash/ || / sh/ || /zsh/ {exit 1;}' && echo "WARNING ** source setup_environment must be run in a bash, zsh or sh shell; see README" export GPGPUSIM_SETUP_ENVIRONMENT_WAS_RUN= export GPGPUSIM_ROOT="$( cd "$( dirname "$BASH_SOURCE" )" && pwd )" diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc index f7f1016..acb376a 100644 --- a/src/abstract_hardware_model.cc +++ b/src/abstract_hardware_model.cc @@ -316,12 +316,12 @@ void warp_inst_t::generate_mem_accesses() break; case global_space: case local_space: case param_space_local: - if( m_config->gpgpu_coalesce_arch == 13 ) { - if(isatomic()) - memory_coalescing_arch_13_atomic(is_write, access_type); - else - memory_coalescing_arch_13(is_write, access_type); - } else abort(); + if( m_config->gpgpu_coalesce_arch >= 13 && m_config->gpgpu_coalesce_arch <= 62) { + if(isatomic()) + memory_coalescing_arch_atomic(is_write, access_type); + else + memory_coalescing_arch(is_write, access_type); + } else abort(); break; @@ -345,7 +345,7 @@ void warp_inst_t::generate_mem_accesses() byte_mask.set(idx+i); } for( a=accesses.begin(); a != accesses.end(); ++a ) - m_accessq.push_back( mem_access_t(access_type,a->first,cache_block_size,is_write,a->second,byte_mask) ); + m_accessq.push_back( mem_access_t(access_type,a->first,cache_block_size,is_write,a->second, byte_mask, mem_access_sector_mask_t())); } if ( space.get_type() == global_space ) { @@ -354,15 +354,32 @@ void warp_inst_t::generate_mem_accesses() m_mem_accesses_created=true; } -void warp_inst_t::memory_coalescing_arch_13( bool is_write, mem_access_type access_type ) +void warp_inst_t::memory_coalescing_arch( bool is_write, mem_access_type access_type ) { // see the CUDA manual where it discusses coalescing rules before reading this unsigned segment_size = 0; unsigned warp_parts = m_config->mem_warp_parts; + bool sector_segment_size = false; + + if(m_config->gpgpu_coalesce_arch >= 20 && m_config->gpgpu_coalesce_arch < 39) + { + //Fermi and Kepler, L1 is normal and L2 is sector + if(m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) + sector_segment_size = true; + else + sector_segment_size = false; + } + else if(m_config->gpgpu_coalesce_arch >= 40) + { + //Maxwell and Pascal, L1 and L2 are sectors + //all requests should be 32 bytes + sector_segment_size = true; + } + switch( data_size ) { case 1: segment_size = 32; break; - case 2: segment_size = 64; break; - case 4: case 8: case 16: segment_size = 128; break; + case 2: segment_size = sector_segment_size? 32 : 64; break; + case 4: case 8: case 16: segment_size = sector_segment_size? 32 : 128; break; } unsigned subwarp_size = m_config->warp_size / warp_parts; @@ -413,13 +430,13 @@ void warp_inst_t::memory_coalescing_arch_13( bool is_write, mem_access_type acce new_addr_type addr = t->first; const transaction_info &info = t->second; - memory_coalescing_arch_13_reduce_and_send(is_write, access_type, info, addr, segment_size); + memory_coalescing_arch_reduce_and_send(is_write, access_type, info, addr, segment_size); } } } -void warp_inst_t::memory_coalescing_arch_13_atomic( bool is_write, mem_access_type access_type ) +void warp_inst_t::memory_coalescing_arch_atomic( bool is_write, mem_access_type access_type ) { assert(space.get_type() == global_space); // Atomics allowed only for global memory @@ -488,13 +505,13 @@ void warp_inst_t::memory_coalescing_arch_13_atomic( bool is_write, mem_access_ty for(t=transaction_list.begin(); t!=transaction_list.end(); t++) { // For each transaction const transaction_info &info = *t; - memory_coalescing_arch_13_reduce_and_send(is_write, access_type, info, addr, segment_size); + memory_coalescing_arch_reduce_and_send(is_write, access_type, info, addr, segment_size); } } } } -void warp_inst_t::memory_coalescing_arch_13_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size ) +void warp_inst_t::memory_coalescing_arch_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size ) { assert( (addr & (segment_size-1)) == 0 ); @@ -543,7 +560,7 @@ void warp_inst_t::memory_coalescing_arch_13_reduce_and_send( bool is_write, mem_ assert(lower_half_used && upper_half_used); } } - m_accessq.push_back( mem_access_t(access_type,addr,size,is_write,info.active,info.bytes) ); + m_accessq.push_back( mem_access_t(access_type,addr,size,is_write,info.active,info.bytes, info.chunks) ); } void warp_inst_t::completed( unsigned long long cycle ) const @@ -577,6 +594,8 @@ kernel_info_t::kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info * //Jin: launch latency management m_launch_latency = g_kernel_launch_latency; + + volta_cache_config_set=false; } kernel_info_t::~kernel_info_t() diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index 71d3d89..a612bac 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -82,6 +82,7 @@ enum uarch_op_t { ALU_OP=1, SFU_OP, TENSOR_CORE_OP, + DP_OP, ALU_SFU_OP, LOAD_OP, TENSOR_CORE_LOAD_OP, @@ -139,6 +140,7 @@ typedef enum special_operations_t special_ops; // Required to identify for the p enum operation_pipeline_t { UNKOWN_OP, SP__OP, + DP__OP, SFU__OP, TENSOR_CORE__OP, MEM__OP @@ -308,6 +310,8 @@ public: unsigned long long start_cycle; unsigned long long end_cycle; unsigned m_launch_latency; + + mutable bool volta_cache_config_set; }; struct core_config { @@ -347,6 +351,7 @@ struct core_config { unsigned gpgpu_cache_constl1_linesize; unsigned gpgpu_max_insn_issue_per_warp; + bool gmem_skip_L1D; // on = global memory access always skip the L1 cache }; // bounded stack that implements simt reconvergence using pdom mechanism from MICRO'07 paper @@ -398,6 +403,7 @@ protected: #define LOCAL_MEM_SIZE_MAX (8*1024) #define MAX_STREAMING_MULTIPROCESSORS 64 #define MAX_THREAD_PER_SM 2048 +#define MAX_WARP_PER_SM 64 #define TOTAL_LOCAL_MEM_PER_SM (MAX_THREAD_PER_SM*LOCAL_MEM_SIZE_MAX) #define TOTAL_SHARED_MEM (MAX_STREAMING_MULTIPROCESSORS*SHARED_MEM_SIZE_MAX) #define TOTAL_LOCAL_MEM (MAX_STREAMING_MULTIPROCESSORS*MAX_THREAD_PER_SM*LOCAL_MEM_SIZE_MAX) @@ -523,7 +529,14 @@ public: const struct textureReference* get_texref(const std::string &texname) const { std::map::const_iterator t=m_NameToTextureRef.find(texname); - assert( t != m_NameToTextureRef.end() ); + if( t == m_NameToTextureRef.end() ) { + // search for :: prefixed names + std::string temp("::" + texname); + t=m_NameToTextureRef.find(temp); + } + + assert(t != m_NameToTextureRef.end()); + return t->second; } const struct cudaArray* get_texarray( const struct textureReference *texref ) const @@ -574,6 +587,7 @@ struct gpgpu_ptx_sim_info int cmem; int gmem; int regs; + unsigned maxthreads; unsigned ptx_version; unsigned sm_target; }; @@ -625,6 +639,9 @@ private: const unsigned MAX_MEMORY_ACCESS_SIZE = 128; typedef std::bitset mem_access_byte_mask_t; +const unsigned SECTOR_CHUNCK_SIZE = 4; //four sectors +const unsigned SECTOR_SIZE = 32 ; //sector is 32 bytes width +typedef std::bitset mem_access_sector_mask_t; #define NO_PARTIAL_WRITE (mem_access_byte_mask_t()) #define MEM_ACCESS_TYPE_TUP_DEF \ @@ -660,6 +677,7 @@ enum cache_operator_type { CACHE_ALL, // .ca CACHE_LAST_USE, // .lu CACHE_VOLATILE, // .cv + CACHE_L1, // .nc // loads and stores CACHE_STREAMING, // .cs @@ -689,8 +707,9 @@ public: unsigned size, bool wr, const active_mask_t &active_mask, - const mem_access_byte_mask_t &byte_mask ) - : m_warp_mask(active_mask), m_byte_mask(byte_mask) + const mem_access_byte_mask_t &byte_mask, + const mem_access_sector_mask_t §or_mask) + : m_warp_mask(active_mask), m_byte_mask(byte_mask), m_sector_mask(sector_mask) { init(); m_type = type; @@ -706,6 +725,7 @@ public: bool is_write() const { return m_write; } enum mem_access_type get_type() const { return m_type; } mem_access_byte_mask_t get_byte_mask() const { return m_byte_mask; } + mem_access_sector_mask_t get_sector_mask() const { return m_sector_mask; } void print(FILE *fp) const { @@ -739,6 +759,7 @@ private: mem_access_type m_type; active_mask_t m_warp_mask; mem_access_byte_mask_t m_byte_mask; + mem_access_sector_mask_t m_sector_mask; static unsigned sm_next_access_uid; }; @@ -961,9 +982,9 @@ public: }; void generate_mem_accesses(); - void memory_coalescing_arch_13( bool is_write, mem_access_type access_type ); - void memory_coalescing_arch_13_atomic( bool is_write, mem_access_type access_type ); - void memory_coalescing_arch_13_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size ); + void memory_coalescing_arch( bool is_write, mem_access_type access_type ); + void memory_coalescing_arch_atomic( bool is_write, mem_access_type access_type ); + void memory_coalescing_arch_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size ); void add_callback( unsigned lane_id, void (*function)(const class inst_t*, class ptx_thread_info*), diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index d5c5c3d..87a0b9c 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -62,8 +62,8 @@ addr_t g_debug_pc = 0xBEEF1518; unsigned g_ptx_sim_num_insn = 0; unsigned gpgpu_param_num_shaders = 0; -char *opcode_latency_int, *opcode_latency_fp, *opcode_latency_dp; -char *opcode_initiation_int, *opcode_initiation_fp, *opcode_initiation_dp; +char *opcode_latency_int, *opcode_latency_fp, *opcode_latency_dp,*opcode_latency_sfu; +char *opcode_initiation_int, *opcode_initiation_fp, *opcode_initiation_dp,*opcode_initiation_sfu; char *cdp_latency_str; unsigned cdp_latency[5]; @@ -80,6 +80,10 @@ void ptx_opcocde_latency_options (option_parser_t opp) { "Opcode latencies for double precision floating points " "Default 8,8,8,8,335", "8,8,8,8,335"); + option_parser_register(opp, "-ptx_opcode_latency_sfu", OPT_CSTR, &opcode_latency_sfu, + "Opcode latencies for SFU instructions" + "Default 8", + "8"); option_parser_register(opp, "-ptx_opcode_initiation_int", OPT_CSTR, &opcode_initiation_int, "Opcode initiation intervals for integers " "Default 1,1,4,4,32", @@ -92,6 +96,10 @@ void ptx_opcocde_latency_options (option_parser_t opp) { "Opcode initiation intervals for double precision floating points " "Default 8,8,8,8,130", "8,8,8,8,130"); + option_parser_register(opp, "-ptx_opcode_initiation_sfu", OPT_CSTR, &opcode_initiation_sfu, + "Opcode initiation intervals for sfu instructions" + "Default 8", + "8"); option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str, "CDP API latency write(dst_start_addr+n,1, src_data+n,NULL,NULL); + + // Copy into the performance model. + extern gpgpu_sim* g_the_gpu; + g_the_gpu->perf_memcpy_to_gpu(dst_start_addr, count); if(g_debug_execution >= 3) { printf( " done.\n"); fflush(stdout); @@ -408,6 +420,10 @@ void gpgpu_t::memcpy_from_gpu( void *dst, size_t src_start_addr, size_t count ) unsigned char *dst_data = (unsigned char*)dst; for (unsigned n=0; n < count; n ++ ) m_global_mem->read(src_start_addr+n,1,dst_data+n); + + // Copy into the performance model. + extern gpgpu_sim* g_the_gpu; + g_the_gpu->perf_memcpy_to_gpu(src_start_addr, count); if(g_debug_execution >= 3) { printf( " done.\n"); fflush(stdout); @@ -592,9 +608,11 @@ void ptx_instruction::set_opcode_and_latency() unsigned int_latency[5]; unsigned fp_latency[5]; unsigned dp_latency[5]; + unsigned sfu_latency; unsigned int_init[5]; unsigned fp_init[5]; unsigned dp_init[5]; + unsigned sfu_init; /* * [0] ADD,SUB * [1] MAX,Min @@ -611,7 +629,9 @@ void ptx_instruction::set_opcode_and_latency() sscanf(opcode_latency_dp, "%u,%u,%u,%u,%u", &dp_latency[0],&dp_latency[1],&dp_latency[2], &dp_latency[3],&dp_latency[4]); - sscanf(opcode_initiation_int, "%u,%u,%u,%u,%u,%u, %u", + sscanf(opcode_latency_sfu, "%u", + &sfu_latency); + sscanf(opcode_initiation_int, "%u,%u,%u,%u,%u", &int_init[0],&int_init[1],&int_init[2], &int_init[3],&int_init[4]); sscanf(opcode_initiation_fp, "%u,%u,%u,%u,%u", @@ -620,8 +640,10 @@ void ptx_instruction::set_opcode_and_latency() sscanf(opcode_initiation_dp, "%u,%u,%u,%u,%u", &dp_init[0],&dp_init[1],&dp_init[2], &dp_init[3],&dp_init[4]); + sscanf(opcode_initiation_sfu, "%u", + &sfu_init); sscanf(cdp_latency_str, "%u,%u,%u,%u,%u", - &cdp_latency[0],&cdp_latency[1],&cdp_latency[2], + &cdp_latency[0],&cdp_latency[1],&cdp_latency[2], &cdp_latency[3],&cdp_latency[4]); if(!m_operands.empty()){ @@ -684,6 +706,7 @@ void ptx_instruction::set_opcode_and_latency() case FF64_TYPE: latency = dp_latency[0]; initiation_interval = dp_init[0]; + op = DP_OP; break; case B32_TYPE: case U32_TYPE: @@ -705,6 +728,7 @@ void ptx_instruction::set_opcode_and_latency() case FF64_TYPE: latency = dp_latency[1]; initiation_interval = dp_init[1]; + op = DP_OP; break; case B32_TYPE: case U32_TYPE: @@ -721,13 +745,12 @@ void ptx_instruction::set_opcode_and_latency() case F32_TYPE: latency = fp_latency[2]; initiation_interval = fp_init[2]; - op = ALU_SFU_OP; break; case F64_TYPE: case FF64_TYPE: latency = dp_latency[2]; initiation_interval = dp_init[2]; - op = ALU_SFU_OP; + op = DP_OP; break; case B32_TYPE: case U32_TYPE: @@ -750,6 +773,7 @@ void ptx_instruction::set_opcode_and_latency() case FF64_TYPE: latency = dp_latency[3]; initiation_interval = dp_init[3]; + op = DP_OP; break; case B32_TYPE: case U32_TYPE: @@ -785,8 +809,8 @@ void ptx_instruction::set_opcode_and_latency() break; case SQRT_OP: case SIN_OP: case COS_OP: case EX2_OP: case LG2_OP: case RSQRT_OP: case RCP_OP: //Using double to approximate those - latency = dp_latency[2]; - initiation_interval = dp_init[2]; + latency = sfu_latency; + initiation_interval = sfu_init; op = SFU_OP; break; case MMA_OP: @@ -796,7 +820,7 @@ void ptx_instruction::set_opcode_and_latency() break; case SHFL_OP: latency = 32; - initiation_interval = 15; + initiation_interval = 4; break; default: break; @@ -889,6 +913,7 @@ void ptx_instruction::pre_decode() switch( m_cache_option ) { case CA_OPTION: cache_op = CACHE_ALL; break; + case NC_OPTION: cache_op = CACHE_L1; break; case CG_OPTION: cache_op = CACHE_GLOBAL; break; case CS_OPTION: cache_op = CACHE_STREAMING; break; case LU_OPTION: cache_op = CACHE_LAST_USE; break; @@ -1169,13 +1194,13 @@ void function_info::finalize( memory_space *param_mem ) // copy the parameter over word-by-word so that parameter that crosses a memory page can be copied over //Jin: copy parameter using aligned rules const size_t word_size = 4; - param_address = (param_address + size - 1) / size * size; //aligned with size + //param_address = (param_address + size - 1) / size * size; //aligned with size TODO: align not correct for (size_t idx = 0; idx < size; idx += word_size) { const char *pdata = reinterpret_cast(param_value.pdata) + idx; // cast to char * for ptr arithmetic param_mem->write(param_address + idx, word_size, pdata,NULL,NULL); } unsigned offset = p.get_offset(); - assert(offset == param_address); + //assert(offset == param_address); param->set_address(param_address); param_address += size; } diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 4614f25..973eeab 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -33,6 +33,7 @@ #include "ptx.tab.h" #include #include +#include #include #include "cuda-math.h" #include "../abstract_hardware_model.h" diff --git a/src/cuda-sim/ptx.y b/src/cuda-sim/ptx.y index 8b901a3..45392fb 100644 --- a/src/cuda-sim/ptx.y +++ b/src/cuda-sim/ptx.y @@ -237,7 +237,8 @@ function_defn: function_decl { set_symtab($1); func_header(".skip"); } statement block_spec: MAXNTID_DIRECTIVE INT_OPERAND COMMA INT_OPERAND COMMA INT_OPERAND {func_header_info_int(".maxntid", $2); func_header_info_int(",", $4); - func_header_info_int(",", $6); } + func_header_info_int(",", $6); + maxnt_id($2, $4, $6);} | MINNCTAPERSM_DIRECTIVE INT_OPERAND { func_header_info_int(".minnctapersm", $2); printf("GPGPU-Sim: Warning: .minnctapersm ignored. \n"); } | MAXNCTAPERSM_DIRECTIVE INT_OPERAND { func_header_info_int(".maxnctapersm", $2); printf("GPGPU-Sim: Warning: .maxnctapersm ignored. \n"); } ; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index d12c741..1813f8c 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -222,6 +222,7 @@ bool symbol_table::add_function_decl( const char *name, int entry_point, functio } else { *func_info = new function_info(entry_point); (*func_info)->set_name(name); + (*func_info)->set_maxnt_id(0); m_function_info_lookup[key] = *func_info; } @@ -1234,16 +1235,16 @@ ptx_instruction::ptx_instruction( int opcode, m_inst_size = 4; // bytes break; case EXTP_OPTION: - break; + break; case NC_OPTION: - break; + m_cache_option = last_ptx_inst_option; + break; case UP_OPTION: case DOWN_OPTION: case BFLY_OPTION: case IDX_OPTION: m_shfl_op = last_ptx_inst_option; break; - case PRMT_F4E_MODE: case PRMT_B4E_MODE: case PRMT_RC8_MODE: diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 0767379..4c36cfc 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1333,6 +1333,7 @@ public: const struct gpgpu_ptx_sim_info* get_kernel_info () const { + assert (m_kernel_info.maxthreads == maxnt_id); return &m_kernel_info; } @@ -1340,6 +1341,8 @@ public: m_kernel_info = info; m_kernel_info.ptx_version = 10*get_ptx_version().ver(); m_kernel_info.sm_target = get_ptx_version().target(); + // THIS DEPENDS ON ptxas being called after the PTX is parsed. + m_kernel_info.maxthreads = maxnt_id; } symbol_table *get_symtab() { @@ -1363,7 +1366,11 @@ public: } bool is_entry_point() const { return m_entry_point; } + void set_maxnt_id(unsigned maxthreads) { maxnt_id = maxthreads;} + unsigned get_maxnt_id() { return maxnt_id;} + private: + unsigned maxnt_id; unsigned m_uid; unsigned m_local_mem_framesize; bool m_entry_point; diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index 9671ab7..cf40365 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -1012,6 +1012,10 @@ void target_header3(char* a, char* b, char* c) g_global_symbol_table->set_sm_target(a,b,c); } +void maxnt_id(int x, int y, int z) { + g_func_info->set_maxnt_id(x * y * z); +} + void func_header(const char* a) {} //intentional dummy function void func_header_info(const char* a) {} //intentional dummy function void func_header_info_int(const char* a, int b) {} //intentional dummy function diff --git a/src/cuda-sim/ptx_parser.h b/src/cuda-sim/ptx_parser.h index 8094b43..7b6e3a2 100644 --- a/src/cuda-sim/ptx_parser.h +++ b/src/cuda-sim/ptx_parser.h @@ -95,6 +95,7 @@ void change_double_operand_type( int addr_type ); void change_operand_neg( ); void set_immediate_operand_type( ); void version_header(double a); +void maxnt_id(int x, int y, int z); //Jin: handle instructino group for cdp void start_inst_group(); diff --git a/src/gpgpu-sim/addrdec.cc b/src/gpgpu-sim/addrdec.cc index 422576d..8651869 100644 --- a/src/gpgpu-sim/addrdec.cc +++ b/src/gpgpu-sim/addrdec.cc @@ -62,6 +62,9 @@ void linear_to_raw_address_translation::addrdec_setoption(option_parser_t opp) option_parser_register(opp, "-gpgpu_mem_address_mask", OPT_INT32, &gpgpu_mem_address_mask, "0 = old addressing mask, 1 = new addressing mask, 2 = new add. mask + flipped bank sel and chip sel bits", "0"); + option_parser_register(opp, "-memory_partition_indexing", OPT_UINT32, &memory_partition_indexing, + "0 = no indexing, 1 = bitwise xoring, 2 = IPoly, 3 = custom indexing", + "0"); } new_addr_type linear_to_raw_address_translation::partition_address( new_addr_type addr ) const @@ -103,6 +106,74 @@ void linear_to_raw_address_translation::addrdec_tlx(new_addr_type addr, addrdec_ tlx->burst= addrdec_packbits(addrdec_mask[BURST], rest_of_addr, addrdec_mkhigh[BURST], addrdec_mklow[BURST]); } + switch(memory_partition_indexing){ + case CONSECUTIVE: + //Do nothing + break; + case BITWISE_PERMUTATION: + { + assert(!gap); + tlx->chip = (tlx->chip) ^ (tlx->row & (m_n_channel-1)); + assert(tlx->chip < m_n_channel); + break; + } + case IPOLY: + { + /* + * Set Indexing function from "Pseudo-randomly interleaved memory." + * Rau, B. R et al. + * ISCA 1991 + * + * equations are adopted from: + * "Sacat: streaming-aware conflict-avoiding thrashing-resistant gpgpu cache management scheme." + * Khairy et al. + * IEEE TPDS 2017. + */ + if(m_n_channel == 32) { + std::bitset<64> a(tlx->row); + std::bitset<5> chip(tlx->chip); + chip[0] = a[13]^a[12]^a[11]^a[10]^a[9]^a[6]^a[5]^a[3]^a[0]^chip[0]; + chip[1] = a[14]^a[13]^a[12]^a[11]^a[10]^a[7]^a[6]^a[4]^a[1]^chip[1]; + chip[2] = a[14]^a[10]^a[9]^a[8]^a[7]^a[6]^a[3]^a[2]^a[0]^chip[2]; + chip[3] = a[11]^a[10]^a[9]^a[8]^a[7]^a[4]^a[3]^a[1]^chip[3]; + chip[4] = a[12]^a[11]^a[10]^a[9]^a[8]^a[5]^a[4]^a[2]^chip[4]; + tlx->chip = chip.to_ulong(); + + } + else{ /* Else incorrect number of channels for the hashing function */ + assert("\nGPGPU-Sim memory_partition_indexing error: The number of channels should be " + "32 for the hashing IPOLY index function.\n" && 0); + } + assert(tlx->chip < m_n_channel); + break; + } + case PAE: + { + //Page Address Entropy + //random selected bits from the page and bank bits + //similar to + //Liu, Yuxi, et al. "Get Out of the Valley: Power-Efficient Address Mapping for GPUs." ISCA 2018 + std::bitset<64> a(tlx->row); + std::bitset<5> chip(tlx->chip); + std::bitset<4> b(tlx->bk); + chip[0] = a[13]^a[10]^a[9]^a[5]^a[0]^b[3]^b[0]^chip[0]; + chip[1] = a[12]^a[11]^a[6]^a[1]^b[3]^b[2]^b[1]^chip[1]; + chip[2] = a[14]^a[9]^a[8]^a[7]^a[2]^b[1]^chip[2]; + chip[3] = a[11]^a[10]^a[8]^a[3]^b[2]^b[3]^chip[3]; + chip[4] = a[12]^a[9]^a[8]^a[5]^a[4]^b[1]^b[0]^chip[4]; + tlx->chip = chip.to_ulong(); + assert(tlx->chip < m_n_channel); + break; + } + case CUSTOM: + /* No custom set function implemented */ + //Do you custom index here + break; + default: + assert("\nUndefined set index function.\n" && 0); + break; + } + // combine the chip address and the lower bits of DRAM bank address to form the subpartition ID unsigned sub_partition_addr_mask = m_n_sub_partition_in_channel - 1; tlx->sub_partition = tlx->chip * m_n_sub_partition_in_channel diff --git a/src/gpgpu-sim/addrdec.h b/src/gpgpu-sim/addrdec.h index fd9af8d..bdc5fec 100644 --- a/src/gpgpu-sim/addrdec.h +++ b/src/gpgpu-sim/addrdec.h @@ -35,6 +35,14 @@ #include "../abstract_hardware_model.h" +enum partition_index_function{ + CONSECUTIVE = 0, + BITWISE_PERMUTATION, + IPOLY, + PAE, + CUSTOM +}; + struct addrdec_t { void print( FILE *fp ) const; @@ -72,6 +80,7 @@ private: const char *addrdec_option; int gpgpu_mem_address_mask; + partition_index_function memory_partition_indexing; bool run_test; int ADDR_CHIP_S; diff --git a/src/gpgpu-sim/delayqueue.h b/src/gpgpu-sim/delayqueue.h index b25f143..0caa5d4 100644 --- a/src/gpgpu-sim/delayqueue.h +++ b/src/gpgpu-sim/delayqueue.h @@ -161,6 +161,7 @@ public: } bool full() const { return (m_max_len && m_length >= m_max_len); } + bool is_avilable_size(unsigned size) const { return (m_max_len && m_length+size-1 >= m_max_len); } bool empty() const { return m_head == NULL; } unsigned get_n_element() const { return m_n_element; } unsigned get_length() const { return m_length; } diff --git a/src/gpgpu-sim/dram.cc b/src/gpgpu-sim/dram.cc index a0e024b..6c11b43 100644 --- a/src/gpgpu-sim/dram.cc +++ b/src/gpgpu-sim/dram.cc @@ -49,11 +49,45 @@ dram_t::dram_t( unsigned int partition_id, const struct memory_config *config, m m_stats = stats; m_config = config; + //rowblp + access_num=0; + hits_num=0; + read_num=0; + write_num=0; + hits_read_num=0; + hits_write_num=0; + banks_1time=0; + banks_acess_total=0; + banks_acess_total_after=0; + banks_time_ready=0; + banks_access_ready_total=0; + issued_two=0; + issued_total=0; + issued_total_row=0; + issued_total_col=0; + CCDc = 0; RRDc = 0; RTWc = 0; WTRc = 0; + wasted_bw_row=0; + wasted_bw_col=0; + util_bw=0; + idle_bw=0; + RCDc_limit=0; + CCDLc_limit=0; + CCDLc_limit_alone=0; + CCDc_limit=0; + WTRc_limit=0; + WTRc_limit_alone=0; + RCDWRc_limit=0; + RTWc_limit=0; + RTWc_limit_alone=0; + rwq_limit=0; + write_to_read_ratio_blp_rw_average=0; + bkgrp_parallsim_rw=0; + rw = READ; //read mode is default bkgrp = (bankgrp_t**) calloc(sizeof(bankgrp_t*), m_config->nbkgrp); @@ -74,12 +108,13 @@ dram_t::dram_t( unsigned int partition_id, const struct memory_config *config, m bk[i]->state = BANK_IDLE; bk[i]->bkgrpindex = i/(m_config->nbk/m_config->nbkgrp); } - prio = 0; + prio = 0; + rwq = new fifo_pipeline("rwq",m_config->CL,m_config->CL+1); mrqq = new fifo_pipeline("mrqq",0,2); returnq = new fifo_pipeline("dramreturnq",0,m_config->gpgpu_dram_return_queue_size==0?1024:m_config->gpgpu_dram_return_queue_size); m_frfcfs_scheduler = NULL; - if ( m_config->scheduler_type == DRAM_FRFCFS ) + if ( m_config->scheduler_type == DRAM_FRFCFS) m_frfcfs_scheduler = new frfcfs_scheduler(m_config,this,stats); n_cmd = 0; n_activity = 0; @@ -88,6 +123,8 @@ dram_t::dram_t( unsigned int partition_id, const struct memory_config *config, m n_pre = 0; n_rd = 0; n_wr = 0; + n_wr_WB=0; + n_rd_L2_A=0; n_req = 0; max_mrqs_temp = 0; bwutil = 0; @@ -113,13 +150,21 @@ dram_t::dram_t( unsigned int partition_id, const struct memory_config *config, m mrqq_Dist = StatCreate("mrqq_length",1, queue_limit()); else //queue length is unlimited; mrqq_Dist = StatCreate("mrqq_length",1,64); //track up to 64 entries + } -bool dram_t::full() const +bool dram_t::full(bool is_write) const { - if(m_config->scheduler_type == DRAM_FRFCFS ){ + if(m_config->scheduler_type == DRAM_FRFCFS){ if(m_config->gpgpu_frfcfs_dram_sched_queue_size == 0 ) return false; - return m_frfcfs_scheduler->num_pending() >= m_config->gpgpu_frfcfs_dram_sched_queue_size; + if(m_config->seperate_write_queue_enabled){ + if(is_write) + return m_frfcfs_scheduler->num_write_pending() >= m_config->gpgpu_frfcfs_dram_write_queue_size; + else + return m_frfcfs_scheduler->num_pending() >= m_config->gpgpu_frfcfs_dram_sched_queue_size; + } + else + return m_frfcfs_scheduler->num_pending() >= m_config->gpgpu_frfcfs_dram_sched_queue_size; } else return mrqq->full(); } @@ -127,7 +172,7 @@ bool dram_t::full() const unsigned dram_t::que_length() const { unsigned nreqs = 0; - if (m_config->scheduler_type == DRAM_FRFCFS ) { + if (m_config->scheduler_type == DRAM_FRFCFS) { nreqs = m_frfcfs_scheduler->num_pending(); } else { nreqs = mrqq->get_length(); @@ -146,7 +191,7 @@ unsigned int dram_t::queue_limit() const } -dram_req_t::dram_req_t( class mem_fetch *mf ) +dram_req_t::dram_req_t( class mem_fetch *mf, unsigned banks, unsigned dram_bnk_indexing_policy) { txbytes = 0; dqbytes = 0; @@ -154,7 +199,29 @@ dram_req_t::dram_req_t( class mem_fetch *mf ) const addrdec_t &tlx = mf->get_tlx_addr(); - bk = tlx.bk; + switch(dram_bnk_indexing_policy){ + case LINEAR_BK_INDEX: + { + bk = tlx.bk; + break; + } + case BITWISE_XORING_BK_INDEX: + { + //xoring bank bits with lower bits of the page + int lbank = log2(banks); + bk = tlx.bk ^ (tlx.row & ((1<get_data_size(); @@ -169,14 +236,15 @@ void dram_t::push( class mem_fetch *data ) { assert(id == data->get_tlx_addr().chip); // Ensure request is in correct memory partition - dram_req_t *mrq = new dram_req_t(data); + dram_req_t *mrq = new dram_req_t(data,m_config->nbk,m_config->dram_bnk_indexing_policy); + data->set_status(IN_PARTITION_MC_INTERFACE_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); - mrqq->push(mrq); + mrqq->push(mrq); // stats... n_req += 1; n_req_partial += 1; - if ( m_config->scheduler_type == DRAM_FRFCFS ) { + if ( m_config->scheduler_type == DRAM_FRFCFS) { unsigned nreqs = m_frfcfs_scheduler->num_pending(); if ( nreqs > max_mrqs_temp) max_mrqs_temp = nreqs; @@ -212,6 +280,7 @@ void dram_t::cycle() printf("\tDQ: BK%d Row:%03x Col:%03x", cmd->bk, cmd->row, cmd->col + cmd->dqbytes); #endif cmd->dqbytes += m_config->dram_atom_size; + if (cmd->dqbytes >= cmd->nbytes) { mem_fetch *data = cmd->data; data->set_status(IN_PARTITION_MC_RETURNQ,gpu_sim_cycle+gpu_tot_sim_cycle); @@ -240,7 +309,7 @@ void dram_t::cycle() printf("Error: Unknown DRAM scheduler type\n"); assert(0); } - if ( m_config->scheduler_type == DRAM_FRFCFS ) { + if ( m_config->scheduler_type == DRAM_FRFCFS) { unsigned nreqs = m_frfcfs_scheduler->num_pending(); if ( nreqs > max_mrqs) { max_mrqs = nreqs; @@ -258,130 +327,123 @@ void dram_t::cycle() unsigned k=m_config->nbk; bool issued = false; - // check if any bank is ready to issue a new read + //collect row buffer locality, BLP and other statistics + ///////////////////////////////////////////////////////////////////////// + unsigned int memory_pending=0; for (unsigned i=0;inbk;i++) { - unsigned j = (i + prio) % m_config->nbk; - unsigned grp = j>>m_config->bk_tag_length; - if (bk[j]->mrq) { //if currently servicing a memory request - bk[j]->mrq->data->set_status(IN_PARTITION_DRAM,gpu_sim_cycle+gpu_tot_sim_cycle); - // correct row activated for a READ - if ( !issued && !CCDc && !bk[j]->RCDc && - !(bkgrp[grp]->CCDLc) && - (bk[j]->curr_row == bk[j]->mrq->row) && - (bk[j]->mrq->rw == READ) && (WTRc == 0 ) && - (bk[j]->state == BANK_ACTIVE) && - !rwq->full() ) { - if (rw==WRITE) { - rw=READ; - rwq->set_min_length(m_config->CL); - } - rwq->push(bk[j]->mrq); - bk[j]->mrq->txbytes += m_config->dram_atom_size; - CCDc = m_config->tCCD; - bkgrp[grp]->CCDLc = m_config->tCCDL; - RTWc = m_config->tRTW; - bk[j]->RTPc = m_config->BL/m_config->data_command_freq_ratio; - bkgrp[grp]->RTPLc = m_config->tRTPL; - issued = true; - n_rd++; - bwutil += m_config->BL/m_config->data_command_freq_ratio; - bwutil_partial += m_config->BL/m_config->data_command_freq_ratio; - bk[j]->n_access++; -#ifdef DRAM_VERIFY - PRINT_CYCLE=1; - printf("\tRD Bk:%d Row:%03x Col:%03x \n", - j, bk[j]->curr_row, - bk[j]->mrq->col + bk[j]->mrq->txbytes - m_config->dram_atom_size); -#endif - // transfer done - if ( !(bk[j]->mrq->txbytes < bk[j]->mrq->nbytes) ) { - bk[j]->mrq = NULL; - } - } else - // correct row activated for a WRITE - if ( !issued && !CCDc && !bk[j]->RCDWRc && - !(bkgrp[grp]->CCDLc) && - (bk[j]->curr_row == bk[j]->mrq->row) && - (bk[j]->mrq->rw == WRITE) && (RTWc == 0 ) && - (bk[j]->state == BANK_ACTIVE) && - !rwq->full() ) { - if (rw==READ) { - rw=WRITE; - rwq->set_min_length(m_config->WL); - } - rwq->push(bk[j]->mrq); - - bk[j]->mrq->txbytes += m_config->dram_atom_size; - CCDc = m_config->tCCD; - bkgrp[grp]->CCDLc = m_config->tCCDL; - WTRc = m_config->tWTR; - bk[j]->WTPc = m_config->tWTP; - issued = true; - n_wr++; - bwutil += m_config->BL/m_config->data_command_freq_ratio; - bwutil_partial += m_config->BL/m_config->data_command_freq_ratio; -#ifdef DRAM_VERIFY - PRINT_CYCLE=1; - printf("\tWR Bk:%d Row:%03x Col:%03x \n", - j, bk[j]->curr_row, - bk[j]->mrq->col + bk[j]->mrq->txbytes - m_config->dram_atom_size); -#endif - // transfer done - if ( !(bk[j]->mrq->txbytes < bk[j]->mrq->nbytes) ) { - bk[j]->mrq = NULL; - } - } - - else - // bank is idle - if ( !issued && !RRDc && - (bk[j]->state == BANK_IDLE) && - !bk[j]->RPc && !bk[j]->RCc ) { -#ifdef DRAM_VERIFY - PRINT_CYCLE=1; - printf("\tACT BK:%d NewRow:%03x From:%03x \n", - j,bk[j]->mrq->row,bk[j]->curr_row); -#endif - // activate the row with current memory request - bk[j]->curr_row = bk[j]->mrq->row; - bk[j]->state = BANK_ACTIVE; - RRDc = m_config->tRRD; - bk[j]->RCDc = m_config->tRCD; - bk[j]->RCDWRc = m_config->tRCDWR; - bk[j]->RASc = m_config->tRAS; - bk[j]->RCc = m_config->tRC; - prio = (j + 1) % m_config->nbk; - issued = true; - n_act_partial++; - n_act++; - } - - else - // different row activated - if ( (!issued) && - (bk[j]->curr_row != bk[j]->mrq->row) && - (bk[j]->state == BANK_ACTIVE) && - (!bk[j]->RASc && !bk[j]->WTPc && - !bk[j]->RTPc && - !bkgrp[grp]->RTPLc) ) { - // make the bank idle again - bk[j]->state = BANK_IDLE; - bk[j]->RPc = m_config->tRP; - prio = (j + 1) % m_config->nbk; - issued = true; - n_pre++; - n_pre_partial++; -#ifdef DRAM_VERIFY - PRINT_CYCLE=1; - printf("\tPRE BK:%d Row:%03x \n", j,bk[j]->curr_row); -#endif - } - } else { - if (!CCDc && !RRDc && !RTWc && !WTRc && !bk[j]->RCDc && !bk[j]->RASc - && !bk[j]->RCc && !bk[j]->RPc && !bk[j]->RCDWRc) k--; - bk[j]->n_idle++; - } + if (bk[i]->mrq) + memory_pending++; } + banks_1time += memory_pending; + if(memory_pending >0) + banks_acess_total++; + + unsigned int memory_pending_rw=0; + unsigned read_blp_rw=0; + unsigned write_blp_rw=0; + std::bitset<8> bnkgrp_rw_found; //assume max we have 8 bank groups + + for (unsigned j=0;jnbk;j++) { + unsigned grp = get_bankgrp_number(j); + if (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == READ) && + (bk[j]->state == BANK_ACTIVE)))) + { + memory_pending_rw++; + read_blp_rw++; + bnkgrp_rw_found.set(grp); + } + else if + (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == WRITE) && + (bk[j]->state == BANK_ACTIVE)))) + { + memory_pending_rw++; + write_blp_rw++; + bnkgrp_rw_found.set(grp); + } + } + banks_time_rw += memory_pending_rw; + bkgrp_parallsim_rw += bnkgrp_rw_found.count(); + if(memory_pending_rw >0) + { + write_to_read_ratio_blp_rw_average += (double)write_blp_rw/(write_blp_rw+read_blp_rw); + banks_access_rw_total++; + } + + unsigned int memory_Pending_ready=0; + for (unsigned j=0;jnbk;j++) { + unsigned grp = get_bankgrp_number(j); + if (bk[j]->mrq && ((!CCDc && !bk[j]->RCDc && + !(bkgrp[grp]->CCDLc) && + (bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == READ) && (WTRc == 0 ) && + (bk[j]->state == BANK_ACTIVE) && + !rwq->full()) + || + (!CCDc && !bk[j]->RCDWRc && + !(bkgrp[grp]->CCDLc) && + (bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == WRITE) && (RTWc == 0 ) && + (bk[j]->state == BANK_ACTIVE) && + !rwq->full()))) + { + memory_Pending_ready++; + } + } + banks_time_ready += memory_Pending_ready; + if(memory_Pending_ready >0) + banks_access_ready_total++; + /////////////////////////////////////////////////////////////////////////////////// + + bool issued_col_cmd = false; + bool issued_row_cmd = false; + + if(m_config->dual_bus_interface) + { + //dual bus interface + //issue one row command and one column command + for (unsigned i=0;inbk;i++) { + unsigned j = (i + prio) % m_config->nbk; + issued_col_cmd = issue_col_command(j); + if(issued_col_cmd) break; + } + for (unsigned i=0;inbk;i++) { + unsigned j = (i + prio) % m_config->nbk; + issued_row_cmd = issue_row_command(j); + if(issued_row_cmd) break; + } + for (unsigned i=0;inbk;i++) { + unsigned j = (i + prio) % m_config->nbk; + if(!bk[j]->mrq) { + if (!CCDc && !RRDc && !RTWc && !WTRc && !bk[j]->RCDc && !bk[j]->RASc + && !bk[j]->RCc && !bk[j]->RPc && !bk[j]->RCDWRc) k--; + bk[j]->n_idle++; + } + } + } + else + { + //single bus interface + //issue only one row/column command + for (unsigned i=0;inbk;i++) { + unsigned j = (i + prio) % m_config->nbk; + if(!issued_col_cmd) + issued_col_cmd = issue_col_command(j); + + if(!issued_col_cmd && !issued_row_cmd) + issued_row_cmd = issue_row_command(j); + + if(!bk[j]->mrq) { + if (!CCDc && !RRDc && !RTWc && !WTRc && !bk[j]->RCDc && !bk[j]->RASc + && !bk[j]->RCc && !bk[j]->RPc && !bk[j]->RCDWRc) k--; + bk[j]->n_idle++; + } + + } + } + + issued = issued_row_cmd || issued_col_cmd; if (!issued) { n_nop++; n_nop_partial++; @@ -395,6 +457,85 @@ void dram_t::cycle() } n_cmd++; n_cmd_partial++; + if(issued) + { + issued_total++; + if(issued_col_cmd && issued_row_cmd) + issued_two++; + } + if(issued_col_cmd) issued_total_col++; + if(issued_row_cmd) issued_total_row++; + + + //Collect some statistics + //check the limitation, see where BW is wasted? + ///////////////////////////////////////////////////////// + unsigned int memory_pending_found=0; + for (unsigned i=0;inbk;i++) { + if (bk[i]->mrq) + memory_pending_found++; + } + if(memory_pending_found>0) + banks_acess_total_after++; + + bool memory_pending_rw_found=false; + for (unsigned j=0;jnbk;j++) { + unsigned grp = get_bankgrp_number(j); + if (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == READ) && + (bk[j]->state == BANK_ACTIVE)) + || + ( + (bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == WRITE) && + (bk[j]->state == BANK_ACTIVE)))) + memory_pending_rw_found=true; + } + + + if(issued_col_cmd || CCDc) + util_bw++; + else if (memory_pending_rw_found) + { + wasted_bw_col++; + for (unsigned j=0;jnbk;j++) { + unsigned grp = get_bankgrp_number(j); + //read + if (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == READ) && + (bk[j]->state == BANK_ACTIVE)))) + { + if(bk[j]->RCDc) RCDc_limit++; + if(bkgrp[grp]->CCDLc) CCDLc_limit++; + if(WTRc) WTRc_limit++; + if(CCDc) CCDc_limit++; + if(rwq->full()) rwq_limit++; + if(bkgrp[grp]->CCDLc && !WTRc) CCDLc_limit_alone++; + if(!bkgrp[grp]->CCDLc && WTRc) WTRc_limit_alone++; + } + //write + else if (bk[j]->mrq && ((bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == WRITE) && + (bk[j]->state == BANK_ACTIVE))) + { + if(bk[j]->RCDWRc) RCDWRc_limit++; + if(bkgrp[grp]->CCDLc) CCDLc_limit++; + if(RTWc) RTWc_limit++; + if(CCDc) CCDc_limit++; + if(rwq->full()) rwq_limit++; + if(bkgrp[grp]->CCDLc && !RTWc) CCDLc_limit_alone++; + if(!bkgrp[grp]->CCDLc && RTWc) RTWc_limit_alone++; + } + } + } + else if (memory_pending_found) + wasted_bw_row++; + else if (!memory_pending_found) + idle_bw++; + else + assert(1); + + ///////////////////////////////////////////////////////// // decrements counters once for each time dram_issueCMD is called DEC2ZERO(RRDc); @@ -420,39 +561,237 @@ void dram_t::cycle() #endif } +bool dram_t::issue_col_command(int j) +{ + bool issued = false; + unsigned grp = get_bankgrp_number(j); + if (bk[j]->mrq) { //if currently servicing a memory request + bk[j]->mrq->data->set_status(IN_PARTITION_DRAM,gpu_sim_cycle+gpu_tot_sim_cycle); + // correct row activated for a READ + if ( !issued && !CCDc && !bk[j]->RCDc && + !(bkgrp[grp]->CCDLc) && + (bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == READ) && (WTRc == 0 ) && + (bk[j]->state == BANK_ACTIVE) && + !rwq->full() ) { + if (rw==WRITE) { + rw=READ; + rwq->set_min_length(m_config->CL); + } + rwq->push(bk[j]->mrq); + bk[j]->mrq->txbytes += m_config->dram_atom_size; + CCDc = m_config->tCCD; + bkgrp[grp]->CCDLc = m_config->tCCDL; + RTWc = m_config->tRTW; + bk[j]->RTPc = m_config->BL/m_config->data_command_freq_ratio; + bkgrp[grp]->RTPLc = m_config->tRTPL; + issued = true; + if(bk[j]->mrq->data->get_access_type() == L2_WR_ALLOC_R) + n_rd_L2_A++; + else + n_rd++; + + bwutil += m_config->BL/m_config->data_command_freq_ratio; + bwutil_partial += m_config->BL/m_config->data_command_freq_ratio; + bk[j]->n_access++; + +#ifdef DRAM_VERIFY + PRINT_CYCLE=1; + printf("\tRD Bk:%d Row:%03x Col:%03x \n", + j, bk[j]->curr_row, + bk[j]->mrq->col + bk[j]->mrq->txbytes - m_config->dram_atom_size); +#endif + // transfer done + if ( !(bk[j]->mrq->txbytes < bk[j]->mrq->nbytes) ) { + bk[j]->mrq = NULL; + } + } else + // correct row activated for a WRITE + if ( !issued && !CCDc && !bk[j]->RCDWRc && + !(bkgrp[grp]->CCDLc) && + (bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == WRITE) && (RTWc == 0 ) && + (bk[j]->state == BANK_ACTIVE) && + !rwq->full() ) { + if (rw==READ) { + rw=WRITE; + rwq->set_min_length(m_config->WL); + } + rwq->push(bk[j]->mrq); + + bk[j]->mrq->txbytes += m_config->dram_atom_size; + CCDc = m_config->tCCD; + bkgrp[grp]->CCDLc = m_config->tCCDL; + WTRc = m_config->tWTR; + bk[j]->WTPc = m_config->tWTP; + issued = true; + + if(bk[j]->mrq->data->get_access_type() == L2_WRBK_ACC) + n_wr_WB++; + else + n_wr++; + bwutil += m_config->BL/m_config->data_command_freq_ratio; + bwutil_partial += m_config->BL/m_config->data_command_freq_ratio; +#ifdef DRAM_VERIFY + PRINT_CYCLE=1; + printf("\tWR Bk:%d Row:%03x Col:%03x \n", + j, bk[j]->curr_row, + bk[j]->mrq->col + bk[j]->mrq->txbytes - m_config->dram_atom_size); +#endif + // transfer done + if ( !(bk[j]->mrq->txbytes < bk[j]->mrq->nbytes) ) { + bk[j]->mrq = NULL; + } + } + + } + + return issued; +} + +bool dram_t::issue_row_command(int j) +{ + bool issued = false; + unsigned grp = get_bankgrp_number(j); + if (bk[j]->mrq) { //if currently servicing a memory request + bk[j]->mrq->data->set_status(IN_PARTITION_DRAM,gpu_sim_cycle+gpu_tot_sim_cycle); + // bank is idle + //else + if ( !issued && !RRDc && + (bk[j]->state == BANK_IDLE) && + !bk[j]->RPc && !bk[j]->RCc) { // +#ifdef DRAM_VERIFY + PRINT_CYCLE=1; + printf("\tACT BK:%d NewRow:%03x From:%03x \n", + j,bk[j]->mrq->row,bk[j]->curr_row); +#endif + // activate the row with current memory request + bk[j]->curr_row = bk[j]->mrq->row; + bk[j]->state = BANK_ACTIVE; + RRDc = m_config->tRRD; + bk[j]->RCDc = m_config->tRCD; + bk[j]->RCDWRc = m_config->tRCDWR; + bk[j]->RASc = m_config->tRAS; + bk[j]->RCc = m_config->tRC; + prio = (j + 1) % m_config->nbk; + issued = true; + n_act_partial++; + n_act++; + } + + else + // different row activated + if ( (!issued) && + (bk[j]->curr_row != bk[j]->mrq->row) && + (bk[j]->state == BANK_ACTIVE) && + (!bk[j]->RASc && !bk[j]->WTPc && + !bk[j]->RTPc && + !bkgrp[grp]->RTPLc) ) { + // make the bank idle again + bk[j]->state = BANK_IDLE; + bk[j]->RPc = m_config->tRP; + prio = (j + 1) % m_config->nbk; + issued = true; + n_pre++; + n_pre_partial++; +#ifdef DRAM_VERIFY + PRINT_CYCLE=1; + printf("\tPRE BK:%d Row:%03x \n", j,bk[j]->curr_row); +#endif + } + } + return issued; +} + + //if mrq is being serviced by dram, gets popped after CL latency fulfilled -class mem_fetch* dram_t::return_queue_pop() +class mem_fetch* dram_t::return_queue_pop() { return returnq->pop(); } -class mem_fetch* dram_t::return_queue_top() +class mem_fetch* dram_t::return_queue_top() { return returnq->top(); } + void dram_t::print( FILE* simFile) const { unsigned i; fprintf(simFile,"DRAM[%d]: %d bks, busW=%d BL=%d CL=%d, ", id, m_config->nbk, m_config->busW, m_config->BL, m_config->CL ); fprintf(simFile,"tRRD=%d tCCD=%d, tRCD=%d tRAS=%d tRP=%d tRC=%d\n", - m_config->tCCD, m_config->tRRD, m_config->tRCD, m_config->tRAS, m_config->tRP, m_config->tRC ); - fprintf(simFile,"n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_req=%d n_rd=%d n_write=%d bw_util=%.4g\n", - n_cmd, n_nop, n_act, n_pre, n_req, n_rd, n_wr, + m_config->tRRD, m_config->tCCD, m_config->tRCD, m_config->tRAS, m_config->tRP, m_config->tRC ); + fprintf(simFile,"n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_ref_event=%d n_req=%d n_rd=%d n_rd_L2_A=%d n_write=%d n_wr_bk=%d bw_util=%.4g\n", + n_cmd, n_nop, n_act, n_pre, n_ref, n_req, n_rd, n_rd_L2_A, n_wr, n_wr_WB, (float)bwutil/n_cmd); fprintf(simFile,"n_activity=%d dram_eff=%.4g\n", n_activity, (float)bwutil/n_activity); for (i=0;inbk;i++) { fprintf(simFile, "bk%d: %da %di ",i,bk[i]->n_access,bk[i]->n_idle); } + fprintf(simFile, "\n"); + fprintf(simFile, "\n------------------------------------------------------------------------\n"); + + printf("\nRow_Buffer_Locality = %.6f", (float)hits_num / access_num); + printf("\nRow_Buffer_Locality_read = %.6f", (float)hits_read_num / read_num); + printf("\nRow_Buffer_Locality_write = %.6f", (float)hits_write_num / write_num); + printf("\nBank_Level_Parallism = %.6f", (float)banks_1time / banks_acess_total); + printf("\nBank_Level_Parallism_Col = %.6f", (float)banks_time_rw / banks_access_rw_total); + printf("\nBank_Level_Parallism_Ready = %.6f", (float)banks_time_ready /banks_access_ready_total); + printf("\nwrite_to_read_ratio_blp_rw_average = %.6f", write_to_read_ratio_blp_rw_average /banks_access_rw_total); + printf("\nGrpLevelPara = %.6f \n", (float)bkgrp_parallsim_rw /banks_access_rw_total); + + printf("\nBW Util details:\n"); + printf("bwutil = %.6f \n", (float)bwutil/n_cmd); + printf("total_CMD = %d \n", n_cmd); + printf("util_bw = %d \n", util_bw); + printf("Wasted_Col = %d \n", wasted_bw_col); + printf("Wasted_Row = %d \n", wasted_bw_row); + printf("Idle = %d \n", idle_bw); + + printf("\nBW Util Bottlenecks: \n"); + printf("RCDc_limit = %d \n", RCDc_limit); + printf("RCDWRc_limit = %d \n", RCDWRc_limit); + printf("WTRc_limit = %d \n", WTRc_limit); + printf("RTWc_limit = %d \n", RTWc_limit); + printf("CCDLc_limit = %d \n", CCDLc_limit); + printf("rwq = %d \n", rwq_limit); + printf("CCDLc_limit_alone = %d \n", CCDLc_limit_alone); + printf("WTRc_limit_alone = %d \n", WTRc_limit_alone); + printf("RTWc_limit_alone = %d \n", RTWc_limit_alone); + + printf("\nCommands details: \n"); + printf("total_CMD = %d \n", n_cmd); + printf("n_nop = %d \n", n_nop); + printf("Read = %d \n", n_rd); + printf("Write = %d \n",n_wr); + printf("L2_Alloc = %d \n", n_rd_L2_A); + printf("L2_WB = %d \n", n_wr_WB); + printf("n_act = %d \n", n_act); + printf("n_pre = %d \n", n_pre); + printf("n_ref = %d \n", n_ref); + printf("n_req = %d \n", n_req ); + printf("total_req = %d \n", n_rd+n_wr+n_rd_L2_A+n_wr_WB); + + printf("\nDual Bus Interface Util: \n"); + printf("issued_total_row = %lu \n", issued_total_row); + printf("issued_total_col = %lu \n", issued_total_col); + printf("Row_Bus_Util = %.6f \n", (float)issued_total_row / n_cmd); + printf("CoL_Bus_Util = %.6f \n", (float)issued_total_col / n_cmd); + printf("Either_Row_CoL_Bus_Util = %.6f \n", (float)issued_total / n_cmd); + printf("Issued_on_Two_Bus_Simul_Util = %.6f \n", (float)issued_two /n_cmd); + printf("issued_two_Eff = %.6f \n", (float)issued_two /issued_total); + printf("queue_avg = %.6f \n\n", (float)ave_mrqs/n_cmd ); + fprintf(simFile, "\n"); fprintf(simFile, "dram_util_bins:"); for (i=0;i<10;i++) fprintf(simFile, " %d", dram_util_bins[i]); fprintf(simFile, "\ndram_eff_bins:"); for (i=0;i<10;i++) fprintf(simFile, " %d", dram_eff_bins[i]); fprintf(simFile, "\n"); - if(m_config->scheduler_type== DRAM_FRFCFS) + if(m_config->scheduler_type== DRAM_FRFCFS) fprintf(simFile, "mrqq: max=%d avg=%g\n", max_mrqs, (float)ave_mrqs/n_cmd); } @@ -476,8 +815,8 @@ void dram_t::visualize() const void dram_t::print_stat( FILE* simFile ) { - fprintf(simFile,"DRAM (%d): n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_req=%d n_rd=%d n_write=%d bw_util=%.4g ", - id, n_cmd, n_nop, n_act, n_pre, n_req, n_rd, n_wr, + fprintf(simFile,"DRAM (%d): n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_ref=%d n_req=%d n_rd=%d n_write=%d bw_util=%.4g ", + id, n_cmd, n_nop, n_act, n_pre, n_ref, n_req, n_rd, n_wr, (float)bwutil/n_cmd); fprintf(simFile, "mrqq: %d %.4g mrqsmax=%d ", max_mrqs, (float)ave_mrqs/n_cmd, max_mrqs_temp); fprintf(simFile, "\n"); @@ -516,6 +855,7 @@ void dram_t::visualizer_print( gzFile visualizer_file ) n_pre_partial = 0; n_req_partial = 0; + // dram access type classification for (unsigned j = 0; j < m_config->nbk; j++) { gzprintf(visualizer_file,"dramglobal_acc_r: %u %u %u\n", id, j, @@ -553,3 +893,16 @@ void dram_t::set_dram_power_stats( unsigned &cmd, wr = n_wr; req = n_req; } + +unsigned dram_t::get_bankgrp_number(unsigned i) +{ + if(m_config->dram_bnkgrp_indexing_policy == HIGHER_BITS) { //higher bits + return i>>m_config->bk_tag_length; + } + else if (m_config->dram_bnkgrp_indexing_policy == LOWER_BITS) { //lower bits + return i&((m_config->nbkgrp-1)); + } + else { + assert(1); + } +} diff --git a/src/gpgpu-sim/dram.h b/src/gpgpu-sim/dram.h index 15c63e7..bee5b7b 100644 --- a/src/gpgpu-sim/dram.h +++ b/src/gpgpu-sim/dram.h @@ -31,9 +31,15 @@ #include "delayqueue.h" #include +#include +#include +#include +#include +#include #include #include #include +#include #define READ 'R' //define read and write states #define WRITE 'W' @@ -42,7 +48,7 @@ class dram_req_t { public: - dram_req_t( class mem_fetch *data ); + dram_req_t( class mem_fetch *data , unsigned banks, unsigned dram_bnk_indexing_policy); unsigned int row; unsigned int col; @@ -87,6 +93,17 @@ struct bank_t unsigned int bkgrpindex; }; +enum bank_index_function{ + LINEAR_BK_INDEX = 0, + BITWISE_XORING_BK_INDEX, + CUSTOM_BK_INDEX +}; + +enum bank_grp_bits_position{ + HIGHER_BITS = 0, + LOWER_BITS +}; + class mem_fetch; class dram_t @@ -95,7 +112,7 @@ public: dram_t( unsigned int parition_id, const struct memory_config *config, class memory_stats_t *stats, class memory_partition_unit *mp ); - bool full() const; + bool full(bool is_write) const; void print( FILE* simFile ) const; void visualize() const; void print_stat( FILE* simFile ); @@ -106,6 +123,7 @@ public: class mem_fetch* return_queue_pop(); class mem_fetch* return_queue_top(); + void push( class mem_fetch *data ); void cycle(); void dram_log (int task); @@ -123,17 +141,24 @@ public: unsigned &wr, unsigned &req) const; -private: - void scheduler_fifo(); - void scheduler_frfcfs(); + const struct memory_config *m_config; +private: bankgrp_t **bkgrp; bank_t **bk; unsigned int prio; + unsigned get_bankgrp_number(unsigned i); + + void scheduler_fifo(); + void scheduler_frfcfs(); + + bool issue_col_command(int j); + bool issue_row_command(int j); + unsigned int RRDc; unsigned int CCDc; unsigned int RTWc; //read to write penalty applies across banks @@ -146,7 +171,7 @@ private: fifo_pipeline *rwq; fifo_pipeline *mrqq; //buffer to hold packets when DRAM processing is over - //should be filled with dram clock and popped with l2or icnt clock + //should be filled with dram clock and popped with l2or icnt clock fifo_pipeline *returnq; unsigned int dram_util_bins[10]; @@ -158,11 +183,51 @@ private: unsigned int n_nop; unsigned int n_act; unsigned int n_pre; + unsigned int n_ref; unsigned int n_rd; + unsigned int n_rd_L2_A; unsigned int n_wr; + unsigned int n_wr_WB; unsigned int n_req; unsigned int max_mrqs_temp; + //some statistics to collect to see where BW is wasted? + unsigned wasted_bw_row; + unsigned wasted_bw_col; + unsigned util_bw; + unsigned idle_bw; + unsigned RCDc_limit; + unsigned CCDLc_limit; + unsigned CCDLc_limit_alone; + unsigned CCDc_limit; + unsigned WTRc_limit; + unsigned WTRc_limit_alone; + unsigned RCDWRc_limit; + unsigned RTWc_limit; + unsigned RTWc_limit_alone; + unsigned rwq_limit; + + //row locality, BLP and other statistics + unsigned long access_num; + unsigned long read_num; + unsigned long write_num; + unsigned long long hits_num; + unsigned long long hits_read_num; + unsigned long long hits_write_num; + unsigned long long banks_1time; + unsigned long long banks_acess_total; + unsigned long long banks_acess_total_after; + unsigned long long banks_time_rw; + unsigned long long banks_access_rw_total; + unsigned long long banks_time_ready; + unsigned long long banks_access_ready_total; + unsigned long long issued_two; + unsigned long long issued_total; + unsigned long long issued_total_row; + unsigned long long issued_total_col; + double write_to_read_ratio_blp_rw_average; + unsigned long long bkgrp_parallsim_rw; + unsigned int bwutil; unsigned int max_mrqs; unsigned int ave_mrqs; diff --git a/src/gpgpu-sim/dram_sched.cc b/src/gpgpu-sim/dram_sched.cc index 8303e86..ff50050 100644 --- a/src/gpgpu-sim/dram_sched.cc +++ b/src/gpgpu-sim/dram_sched.cc @@ -36,6 +36,7 @@ frfcfs_scheduler::frfcfs_scheduler( const memory_config *config, dram_t *dm, mem m_config = config; m_stats = stats; m_num_pending = 0; + m_num_write_pending = 0; m_dram = dm; m_queue = new std::list[m_config->nbk]; m_bins = new std::map::iterator> >[ m_config->nbk ]; @@ -49,15 +50,36 @@ frfcfs_scheduler::frfcfs_scheduler( const memory_config *config, dram_t *dm, mem curr_row_service_time[i] = 0; row_service_timestamp[i] = 0; } + if(m_config->seperate_write_queue_enabled) { + m_write_queue = new std::list[m_config->nbk]; + m_write_bins = new std::map::iterator> >[ m_config->nbk ]; + m_last_write_row = new std::list::iterator>*[ m_config->nbk ]; + + for ( unsigned i=0; i < m_config->nbk; i++ ) { + m_write_queue[i].clear(); + m_write_bins[i].clear(); + m_last_write_row[i] = NULL; + } + } + m_mode = READ_MODE; } void frfcfs_scheduler::add_req( dram_req_t *req ) { - m_num_pending++; - m_queue[req->bk].push_front(req); - std::list::iterator ptr = m_queue[req->bk].begin(); - m_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front + if(m_config->seperate_write_queue_enabled && req->data->is_write()) { + assert(m_num_write_pending < m_config->gpgpu_frfcfs_dram_write_queue_size); + m_num_write_pending++; + m_write_queue[req->bk].push_front(req); + std::list::iterator ptr = m_write_queue[req->bk].begin(); + m_write_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front + } else { + assert(m_num_pending < m_config->gpgpu_frfcfs_dram_sched_queue_size); + m_num_pending++; + m_queue[req->bk].push_front(req); + std::list::iterator ptr = m_queue[req->bk].begin(); + m_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front + } } void frfcfs_scheduler::data_collection(unsigned int bank) @@ -78,41 +100,92 @@ void frfcfs_scheduler::data_collection(unsigned int bank) dram_req_t *frfcfs_scheduler::schedule( unsigned bank, unsigned curr_row ) { - if ( m_last_row[bank] == NULL ) { - if ( m_queue[bank].empty() ) + //row + bool rowhit = true; + std::list *m_current_queue = m_queue; + std::map::iterator> > *m_current_bins = m_bins ; + std::list::iterator> **m_current_last_row = m_last_row; + + if(m_config->seperate_write_queue_enabled) { + if(m_mode == READ_MODE && + ((m_num_write_pending >= m_config->write_high_watermark ) + // || (m_queue[bank].empty() && !m_write_queue[bank].empty()) + )) { + m_mode = WRITE_MODE; + } + else if(m_mode == WRITE_MODE && + (( m_num_write_pending < m_config->write_low_watermark ) + // || (!m_queue[bank].empty() && m_write_queue[bank].empty()) + )){ + m_mode = READ_MODE; + } + } + + if(m_mode == WRITE_MODE) { + m_current_queue = m_write_queue; + m_current_bins = m_write_bins ; + m_current_last_row = m_last_write_row; + } + + if ( m_current_last_row[bank] == NULL ) { + if ( m_current_queue[bank].empty() ) return NULL; - std::map::iterator> >::iterator bin_ptr = m_bins[bank].find( curr_row ); - if ( bin_ptr == m_bins[bank].end()) { - dram_req_t *req = m_queue[bank].back(); - bin_ptr = m_bins[bank].find( req->row ); - assert( bin_ptr != m_bins[bank].end() ); // where did the request go??? - m_last_row[bank] = &(bin_ptr->second); + std::map::iterator> >::iterator bin_ptr = m_current_bins[bank].find( curr_row ); + if ( bin_ptr == m_current_bins[bank].end()) { + dram_req_t *req = m_current_queue[bank].back(); + bin_ptr = m_current_bins[bank].find( req->row ); + assert( bin_ptr != m_current_bins[bank].end() ); // where did the request go??? + m_current_last_row[bank] = &(bin_ptr->second); data_collection(bank); + rowhit = false; } else { - m_last_row[bank] = &(bin_ptr->second); - + m_current_last_row[bank] = &(bin_ptr->second); + rowhit = true; } } - std::list::iterator next = m_last_row[bank]->back(); + std::list::iterator next = m_current_last_row[bank]->back(); dram_req_t *req = (*next); + //rowblp stats + m_dram->access_num++; + bool is_write = req->data->is_write(); + if(is_write) + m_dram->write_num++; + else + m_dram->read_num++; + + if(rowhit) { + m_dram->hits_num++; + if(is_write) + m_dram->hits_write_num++; + else + m_dram->hits_read_num++; + } + m_stats->concurrent_row_access[m_dram->id][bank]++; m_stats->row_access[m_dram->id][bank]++; - m_last_row[bank]->pop_back(); + m_current_last_row[bank]->pop_back(); - m_queue[bank].erase(next); - if ( m_last_row[bank]->empty() ) { - m_bins[bank].erase( req->row ); - m_last_row[bank] = NULL; + m_current_queue[bank].erase(next); + if ( m_current_last_row[bank]->empty() ) { + m_current_bins[bank].erase( req->row ); + m_current_last_row[bank] = NULL; } #ifdef DEBUG_FAST_IDEAL_SCHED if ( req ) printf("%08u : DRAM(%u) scheduling memory request to bank=%u, row=%u\n", (unsigned)gpu_sim_cycle, m_dram->id, req->bk, req->row ); #endif - assert( req != NULL && m_num_pending != 0 ); - m_num_pending--; + + if(m_config->seperate_write_queue_enabled && req->data->is_write()) { + assert( req != NULL && m_num_write_pending != 0 ); + m_num_write_pending--; + } + else { + assert( req != NULL && m_num_pending != 0 ); + m_num_pending--; + } return req; } @@ -129,7 +202,7 @@ void dram_t::scheduler_frfcfs() { unsigned mrq_latency; frfcfs_scheduler *sched = m_frfcfs_scheduler; - while ( !mrqq->empty() && (!m_config->gpgpu_frfcfs_dram_sched_queue_size || sched->num_pending() < m_config->gpgpu_frfcfs_dram_sched_queue_size)) { + while ( !mrqq->empty() ) { dram_req_t *req = mrqq->pop(); // Power stats @@ -160,6 +233,8 @@ void dram_t::scheduler_frfcfs() bk[b]->mrq = req; if (m_config->gpgpu_memlatency_stat) { mrq_latency = gpu_sim_cycle + gpu_tot_sim_cycle - bk[b]->mrq->timestamp; + m_stats->tot_mrq_latency += mrq_latency; + m_stats->tot_mrq_num++; bk[b]->mrq->timestamp = gpu_tot_sim_cycle + gpu_sim_cycle; m_stats->mrq_lat_table[LOGB2(mrq_latency)]++; if (mrq_latency > m_stats->max_mrq_latency) { diff --git a/src/gpgpu-sim/dram_sched.h b/src/gpgpu-sim/dram_sched.h index 3860f5b..63f5831 100644 --- a/src/gpgpu-sim/dram_sched.h +++ b/src/gpgpu-sim/dram_sched.h @@ -35,6 +35,11 @@ #include #include +enum memory_mode { + READ_MODE = 0, + WRITE_MODE +}; + class frfcfs_scheduler { public: frfcfs_scheduler( const memory_config *config, dram_t *dm, memory_stats_t *stats ); @@ -43,17 +48,24 @@ public: dram_req_t *schedule( unsigned bank, unsigned curr_row ); void print( FILE *fp ); unsigned num_pending() const { return m_num_pending;} + unsigned num_write_pending() const { return m_num_write_pending;} private: const memory_config *m_config; dram_t *m_dram; unsigned m_num_pending; + unsigned m_num_write_pending; std::list *m_queue; std::map::iterator> > *m_bins; std::list::iterator> **m_last_row; unsigned *curr_row_service_time; //one set of variables for each bank. unsigned *row_service_timestamp; //tracks when scheduler began servicing current row + std::list *m_write_queue; + std::map::iterator> > *m_write_bins; + std::list::iterator> **m_last_write_row; + + enum memory_mode m_mode; memory_stats_t *m_stats; }; diff --git a/src/gpgpu-sim/gpu-cache.cc b/src/gpgpu-sim/gpu-cache.cc index 8886398..ba81440 100644 --- a/src/gpgpu-sim/gpu-cache.cc +++ b/src/gpgpu-sim/gpu-cache.cc @@ -29,7 +29,6 @@ #include "stat-tool.h" #include -#define MAX_DEFAULT_CACHE_SIZE_MULTIBLIER 4 // used to allocate memory that is large enough to adapt the changes in cache size across kernels const char * cache_request_status_str(enum cache_request_status status) @@ -38,7 +37,8 @@ const char * cache_request_status_str(enum cache_request_status status) "HIT", "HIT_RESERVED", "MISS", - "RESERVATION_FAIL" + "RESERVATION_FAIL", + "SECTOR_MISS" }; assert(sizeof(static_cache_request_status_str) / sizeof(const char*) == NUM_CACHE_REQUEST_STATUS); @@ -47,6 +47,22 @@ const char * cache_request_status_str(enum cache_request_status status) return static_cache_request_status_str[status]; } +const char * cache_fail_status_str(enum cache_reservation_fail_reason status) +{ + static const char * static_cache_reservation_fail_reason_str[] = { + "LINE_ALLOC_FAIL", + "MISS_QUEUE_FULL", + "MSHR_ENRTY_FAIL", + "MSHR_MERGE_ENRTY_FAIL", + "MSHR_RW_PENDING" + }; + + assert(sizeof(static_cache_reservation_fail_reason_str) / sizeof(const char*) == NUM_CACHE_RESERVATION_FAIL_STATUS); + assert(status < NUM_CACHE_RESERVATION_FAIL_STATUS); + + return static_cache_reservation_fail_reason_str[status]; +} + unsigned l1d_cache_config::set_index(new_addr_type addr) const{ unsigned set_index = m_nset; // Default to linear set index function unsigned lower_xor = 0; @@ -54,10 +70,11 @@ unsigned l1d_cache_config::set_index(new_addr_type addr) const{ switch(m_set_index_function){ case FERMI_HASH_SET_FUNCTION: + case BITWISE_XORING_FUNCTION: /* * Set Indexing function from "A Detailed GPU Cache Model Based on Reuse Distance Theory" * Cedric Nugteren et al. - * ISCA 2014 + * HPCA 2014 */ if(m_nset == 32 || m_nset == 64){ // Lower xor value is bits 7-11 @@ -80,6 +97,36 @@ unsigned l1d_cache_config::set_index(new_addr_type addr) const{ } break; + case HASH_IPOLY_FUNCTION: + /* + * Set Indexing function from "Pseudo-randomly interleaved memory." + * Rau, B. R et al. + * ISCA 1991 + * + * "Sacat: streaming-aware conflict-avoiding thrashing-resistant gpgpu cache management scheme." + * Khairy et al. + * IEEE TPDS 2017. + */ + if(m_nset == 32 || m_nset == 64){ + std::bitset<64> a(addr); + std::bitset<6> index; + index[0] = a[25]^a[24]^a[23]^a[22]^a[21]^a[18]^a[17]^a[15]^a[12]^a[7]; //10 + index[1] = a[26]^a[25]^a[24]^a[23]^a[22]^a[19]^a[18]^a[16]^a[13]^a[8]; //10 + index[2] = a[26]^a[22]^a[21]^a[20]^a[19]^a[18]^a[15]^a[14]^a[12]^a[9]; //10 + index[3] = a[23]^a[22]^a[21]^a[20]^a[19]^a[16]^a[15]^a[13]^a[10]; //9 + index[4] = a[24]^a[23]^a[22]^a[21]^a[20]^a[17]^a[16]^a[14]^a[11]; //9 + + if(m_nset == 64) + index[5] = a[12]; + + set_index = index.to_ulong(); + + }else{ /* Else incorrect number of sets for the hashing function */ + assert("\nGPGPU-Sim cache configuration error: The number of sets should be " + "32 or 64 for the hashing set index function.\n" && 0); + } + break; + case CUSTOM_SET_FUNCTION: /* No custom set function implemented */ break; @@ -87,6 +134,10 @@ unsigned l1d_cache_config::set_index(new_addr_type addr) const{ case LINEAR_SET_FUNCTION: set_index = (addr >> m_line_sz_log2) & (m_nset-1); break; + + default: + assert("\nUndefined set index function.\n" && 0); + break; } // Linear function selected or custom set index function not implemented @@ -113,13 +164,16 @@ unsigned l2_cache_config::set_index(new_addr_type addr) const{ tag_array::~tag_array() { + unsigned cache_lines_num = m_config.get_max_num_lines(); + for(unsigned i=0; iget_addr()); + line_table::const_iterator i = pending_lines.find(addr); + if ( i == pending_lines.end() ) { + pending_lines[addr] = mf->get_inst().get_uid(); + } +} + +void tag_array::remove_pending_line(mem_fetch *mf){ + assert(mf); + new_addr_type addr = m_config.block_addr(mf->get_addr()); + line_table::const_iterator i = pending_lines.find(addr); + if ( i != pending_lines.end() ) { + pending_lines.erase(addr); + } } -enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx ) const { +enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx, mem_fetch* mf, bool probe_mode) const { + mem_access_sector_mask_t mask = mf->get_access_sector_mask(); + return probe(addr, idx, mask, probe_mode, mf); +} + + +enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx, mem_access_sector_mask_t mask, bool probe_mode, mem_fetch* mf) const { //assert( m_config.m_write_policy == READ_ONLY ); unsigned set_index = m_config.set_index(addr); new_addr_type tag = m_config.tag(addr); @@ -169,35 +263,45 @@ enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx ) // check for hit or pending hit for (unsigned way=0; waym_tag == tag) { - if ( line->m_status == RESERVED ) { + if ( line->get_status(mask) == RESERVED ) { idx = index; return HIT_RESERVED; - } else if ( line->m_status == VALID ) { + } else if ( line->get_status(mask) == VALID ) { idx = index; return HIT; - } else if ( line->m_status == MODIFIED ) { + } else if ( line->get_status(mask) == MODIFIED) { + if(line->is_readable(mask)) { + idx = index; + return HIT; + } + else { + idx = index; + return SECTOR_MISS; + } + + } else if ( line->is_valid_line() && line->get_status(mask) == INVALID ) { idx = index; - return HIT; - } else { - assert( line->m_status == INVALID ); + return SECTOR_MISS; + }else { + assert( line->get_status(mask) == INVALID ); } } - if (line->m_status != RESERVED) { + if (!line->is_reserved_line()) { all_reserved = false; - if (line->m_status == INVALID) { + if (line->is_invalid_line()) { invalid_line = index; } else { // valid line : keep track of most appropriate replacement candidate if ( m_config.m_replacement_policy == LRU ) { - if ( line->m_last_access_time < valid_timestamp ) { - valid_timestamp = line->m_last_access_time; + if ( line->get_last_access_time() < valid_timestamp ) { + valid_timestamp = line->get_last_access_time(); valid_line = index; } } else if ( m_config.m_replacement_policy == FIFO ) { - if ( line->m_alloc_time < valid_timestamp ) { - valid_timestamp = line->m_alloc_time; + if ( line->get_alloc_time() < valid_timestamp ) { + valid_timestamp = line->get_alloc_time(); valid_line = index; } } @@ -215,40 +319,59 @@ enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx ) idx = valid_line; } else abort(); // if an unreserved block exists, it is either invalid or replaceable + + if(probe_mode && m_config.is_streaming()){ + line_table::const_iterator i = pending_lines.find(m_config.block_addr(addr)); + assert(mf); + if ( !mf->is_write() && i != pending_lines.end() ) { + if(i->second != mf->get_inst().get_uid()) + return SECTOR_MISS; + } + } + return MISS; } -enum cache_request_status tag_array::access( new_addr_type addr, unsigned time, unsigned &idx ) +enum cache_request_status tag_array::access( new_addr_type addr, unsigned time, unsigned &idx, mem_fetch* mf) { bool wb=false; - cache_block_t evicted; - enum cache_request_status result = access(addr,time,idx,wb,evicted); + evicted_block_info evicted; + enum cache_request_status result = access(addr,time,idx,wb,evicted,mf); assert(!wb); return result; } -enum cache_request_status tag_array::access( new_addr_type addr, unsigned time, unsigned &idx, bool &wb, cache_block_t &evicted ) +enum cache_request_status tag_array::access( new_addr_type addr, unsigned time, unsigned &idx, bool &wb, evicted_block_info &evicted, mem_fetch* mf ) { m_access++; + is_used = true; shader_cache_access_log(m_core_id, m_type_id, 0); // log accesses to cache - enum cache_request_status status = probe(addr,idx); + enum cache_request_status status = probe(addr,idx,mf); switch (status) { case HIT_RESERVED: m_pending_hit++; case HIT: - m_lines[idx].m_last_access_time=time; + m_lines[idx]->set_last_access_time(time, mf->get_access_sector_mask()); break; case MISS: m_miss++; shader_cache_access_log(m_core_id, m_type_id, 1); // log cache misses if ( m_config.m_alloc_policy == ON_MISS ) { - if( m_lines[idx].m_status == MODIFIED ) { + if( m_lines[idx]->is_modified_line()) { wb = true; - evicted = m_lines[idx]; + evicted.set_info(m_lines[idx]->m_block_addr, m_lines[idx]->get_modified_size()); } - m_lines[idx].allocate( m_config.tag(addr), m_config.block_addr(addr), time ); + m_lines[idx]->allocate( m_config.tag(addr), m_config.block_addr(addr), time, mf->get_access_sector_mask()); } break; + case SECTOR_MISS: + assert(m_config.m_cache_type == SECTOR); + m_sector_miss++; + shader_cache_access_log(m_core_id, m_type_id, 1); // log cache misses + if ( m_config.m_alloc_policy == ON_MISS ) { + ((sector_cache_block*)m_lines[idx])->allocate_sector( time, mf->get_access_sector_mask() ); + } + break; case RESERVATION_FAIL: m_res_fail++; shader_cache_access_log(m_core_id, m_type_id, 1); // log cache misses @@ -261,37 +384,70 @@ enum cache_request_status tag_array::access( new_addr_type addr, unsigned time, return status; } -void tag_array::fill( new_addr_type addr, unsigned time ) +void tag_array::fill( new_addr_type addr, unsigned time, mem_fetch* mf) +{ + fill(addr, time, mf->get_access_sector_mask()); +} + +void tag_array::fill( new_addr_type addr, unsigned time, mem_access_sector_mask_t mask ) { - assert( m_config.m_alloc_policy == ON_FILL ); + //assert( m_config.m_alloc_policy == ON_FILL ); unsigned idx; - enum cache_request_status status = probe(addr,idx); - assert(status==MISS); // MSHR should have prevented redundant memory request - m_lines[idx].allocate( m_config.tag(addr), m_config.block_addr(addr), time ); - m_lines[idx].fill(time); + enum cache_request_status status = probe(addr,idx,mask); + //assert(status==MISS||status==SECTOR_MISS); // MSHR should have prevented redundant memory request + if(status==MISS) + m_lines[idx]->allocate( m_config.tag(addr), m_config.block_addr(addr), time, mask ); + else if (status==SECTOR_MISS) { + assert(m_config.m_cache_type == SECTOR); + ((sector_cache_block*)m_lines[idx])->allocate_sector( time, mask ); + } + + m_lines[idx]->fill(time, mask); } -void tag_array::fill( unsigned index, unsigned time ) +void tag_array::fill( unsigned index, unsigned time, mem_fetch* mf) { assert( m_config.m_alloc_policy == ON_MISS ); - m_lines[index].fill(time); + m_lines[index]->fill(time, mf->get_access_sector_mask()); } + +//TODO: we need write back the flushed data to the upper level void tag_array::flush() { + if(!is_used) + return; + for (unsigned i=0; i < m_config.get_num_lines(); i++) - m_lines[i].m_status = INVALID; + if(m_lines[i]->is_modified_line()) { + for(unsigned j=0; j < SECTOR_CHUNCK_SIZE; j++) + m_lines[i]->set_status(INVALID, mem_access_sector_mask_t().set(j)) ; + } + + is_used = false; +} + +void tag_array::invalidate() +{ + if(!is_used) + return; + + for (unsigned i=0; i < m_config.get_num_lines(); i++) + for(unsigned j=0; j < SECTOR_CHUNCK_SIZE; j++) + m_lines[i]->set_status(INVALID, mem_access_sector_mask_t().set(j)) ; + + is_used = false; } float tag_array::windowed_miss_rate( ) const { unsigned n_access = m_access - m_prev_snapshot_access; - unsigned n_miss = m_miss - m_prev_snapshot_miss; + unsigned n_miss = (m_miss+m_sector_miss) - m_prev_snapshot_miss; // unsigned n_pending_hit = m_pending_hit - m_prev_snapshot_pending_hit; float missrate = 0.0f; if (n_access != 0) - missrate = (float) n_miss / n_access; + missrate = (float) (n_miss+m_sector_miss) / n_access; return missrate; } @@ -299,23 +455,24 @@ void tag_array::new_window() { m_prev_snapshot_access = m_access; m_prev_snapshot_miss = m_miss; + m_prev_snapshot_miss = m_miss + m_sector_miss; m_prev_snapshot_pending_hit = m_pending_hit; } void tag_array::print( FILE *stream, unsigned &total_access, unsigned &total_misses ) const { m_config.print(stream); - fprintf( stream, "\t\tAccess = %d, Miss = %d (%.3g), PendingHit = %d (%.3g)\n", - m_access, m_miss, (float) m_miss / m_access, + fprintf( stream, "\t\tAccess = %d, Miss = %d, Sector_Miss = %d, Total_Miss = %d (%.3g), PendingHit = %d (%.3g)\n", + m_access, m_miss, m_sector_miss, (m_miss+m_sector_miss), (float) (m_miss+m_sector_miss) / m_access, m_pending_hit, (float) m_pending_hit / m_access); - total_misses+=m_miss; + total_misses+=(m_miss+m_sector_miss); total_access+=m_access; } void tag_array::get_stats(unsigned &total_access, unsigned &total_misses, unsigned &total_hit_res, unsigned &total_res_fail) const{ // Update statistics from the tag array total_access = m_access; - total_misses = m_miss; + total_misses = (m_miss+m_sector_miss); total_hit_res = m_pending_hit; total_res_fail = m_res_fail; } @@ -324,16 +481,17 @@ void tag_array::get_stats(unsigned &total_access, unsigned &total_misses, unsign bool was_write_sent( const std::list &events ) { for( std::list::const_iterator e=events.begin(); e!=events.end(); e++ ) { - if( *e == WRITE_REQUEST_SENT ) + if( (*e).m_cache_event_type == WRITE_REQUEST_SENT ) return true; } return false; } -bool was_writeback_sent( const std::list &events ) +bool was_writeback_sent( const std::list &events, cache_event& wb_event) { for( std::list::const_iterator e=events.begin(); e!=events.end(); e++ ) { - if( *e == WRITE_BACK_REQUEST_SENT ) + if( (*e).m_cache_event_type == WRITE_BACK_REQUEST_SENT ) + wb_event = *e; return true; } return false; @@ -342,7 +500,16 @@ bool was_writeback_sent( const std::list &events ) bool was_read_sent( const std::list &events ) { for( std::list::const_iterator e=events.begin(); e!=events.end(); e++ ) { - if( *e == READ_REQUEST_SENT ) + if( (*e).m_cache_event_type == READ_REQUEST_SENT ) + return true; + } + return false; +} + +bool was_writeallocate_sent( const std::list &events ) +{ + for( std::list::const_iterator e=events.begin(); e!=events.end(); e++ ) { + if( (*e).m_cache_event_type == WRITE_ALLOCATE_SENT ) return true; } return false; @@ -375,11 +542,27 @@ void mshr_table::add( new_addr_type block_addr, mem_fetch *mf ){ } } +/// check is_read_after_write_pending +bool mshr_table::is_read_after_write_pending( new_addr_type block_addr){ + std::list my_list = m_data[block_addr].m_list; + bool write_found = false; + for (std::list::iterator it=my_list.begin(); it != my_list.end(); ++it) + { + if((*it)->is_write()) //Pending Write Request + write_found = true; + else if(write_found) //Pending Read Request and we found previous Write + return true; + } + + return false; + +} + /// Accept a new cache fill response: mark entry ready for processing void mshr_table::mark_ready( new_addr_type block_addr, bool &has_atomic ){ assert( !busy() ); table::iterator a = m_data.find(block_addr); - assert( a != m_data.end() ); // don't remove same request twice + assert( a != m_data.end() ); m_current_response.push_back( block_addr ); has_atomic = a->second.m_has_atomic; assert( m_current_response.size() <= m_data.size() ); @@ -417,9 +600,11 @@ void mshr_table::display( FILE *fp ) const{ /***************************************************************** Caches *****************************************************************/ cache_stats::cache_stats(){ m_stats.resize(NUM_MEM_ACCESS_TYPE); + m_fail_stats.resize(NUM_MEM_ACCESS_TYPE); for(unsigned i=0; i 0) - fprintf(fout, "\t%s[%s][%s] = %u\n", - m_cache_name.c_str(), - mem_access_type_str((enum mem_access_type)type), - "TOTAL_ACCESS", - total_access[type]); + if(total_access[type] > 0) + fprintf(fout, "\t%s[%s][%s] = %u\n", + m_cache_name.c_str(), + mem_access_type_str((enum mem_access_type)type), + "TOTAL_ACCESS", + total_access[type]); } } +void cache_stats::print_fail_stats(FILE *fout, const char *cache_name) const{ + std::string m_cache_name = cache_name; + for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) { + for (unsigned fail = 0; fail < NUM_CACHE_RESERVATION_FAIL_STATUS; ++fail) { + if(m_fail_stats[type][fail] > 0){ + fprintf(fout, "\t%s[%s][%s] = %u\n", + m_cache_name.c_str(), + mem_access_type_str((enum mem_access_type)type), + cache_fail_status_str((enum cache_reservation_fail_reason)fail), + m_fail_stats[type][fail]); + } + } + } +} + void cache_sub_stats::print_port_stats(FILE *fout, const char *cache_name) const { float data_port_util = 0.0f; @@ -580,10 +813,10 @@ void cache_stats::get_sub_stats(struct cache_sub_stats &css) const{ for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) { for (unsigned status = 0; status < NUM_CACHE_REQUEST_STATUS; ++status) { - if(status == HIT || status == MISS || status == HIT_RESERVED) + if(status == HIT || status == MISS || status == SECTOR_MISS || status == HIT_RESERVED) t_css.accesses += m_stats[type][status]; - if(status == MISS) + if(status == MISS || status == SECTOR_MISS) t_css.misses += m_stats[type][status]; if(status == HIT_RESERVED) @@ -611,6 +844,16 @@ bool cache_stats::check_valid(int type, int status) const{ return false; } +bool cache_stats::check_fail_valid(int type, int fail) const{ + /// + /// Verify a valid access_type/access_status + /// + if((type >= 0) && (type < NUM_MEM_ACCESS_TYPE) && (fail >= 0) && (fail < NUM_CACHE_RESERVATION_FAIL_STATUS)) + return true; + else + return false; +} + void cache_stats::sample_cache_port_utility(bool data_port_busy, bool fill_port_busy) { m_cache_port_available_cycles += 1; @@ -639,15 +882,18 @@ void baseline_cache::bandwidth_management::use_data_port(mem_fetch *mf, enum cac unsigned data_cycles = data_size / port_width + ((data_size % port_width > 0)? 1 : 0); m_data_port_occupied_cycles += data_cycles; } break; - case HIT_RESERVED: + case HIT_RESERVED: case MISS: { // the data array is accessed to read out the entire line for write-back - if (was_writeback_sent(events)) { - unsigned data_cycles = m_config.m_line_sz / port_width; + // in case of sector cache we need to write bank only the modified sectors + cache_event ev(WRITE_BACK_REQUEST_SENT); + if (was_writeback_sent(events, ev)) { + unsigned data_cycles = ev.m_evicted_block.m_modified_size / port_width; m_data_port_occupied_cycles += data_cycles; } } break; - case RESERVATION_FAIL: + case SECTOR_MISS: + case RESERVATION_FAIL: // Does not consume any port bandwidth break; default: @@ -660,7 +906,7 @@ void baseline_cache::bandwidth_management::use_data_port(mem_fetch *mf, enum cac void baseline_cache::bandwidth_management::use_fill_port(mem_fetch *mf) { // assume filling the entire line with the returned request - unsigned fill_cycles = m_config.m_line_sz / m_config.m_data_port_width; + unsigned fill_cycles = m_config.get_atom_sz() / m_config.m_data_port_width; m_fill_port_occupied_cycles += fill_cycles; } @@ -707,21 +953,43 @@ void baseline_cache::cycle(){ /// Interface for response from lower memory level (model bandwidth restictions in caller) void baseline_cache::fill(mem_fetch *mf, unsigned time){ + + if(m_config.m_mshr_type == SECTOR_ASSOC) { + assert(mf->get_original_mf()); + extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf->get_original_mf()); + assert( e != m_extra_mf_fields.end() ); + e->second.pending_read--; + + if(e->second.pending_read > 0) { + //wait for the other requests to come back + delete mf; + return; + } else { + mem_fetch *temp = mf; + mf = mf->get_original_mf(); + delete temp; + } + } + extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf); assert( e != m_extra_mf_fields.end() ); assert( e->second.m_valid ); mf->set_data_size( e->second.m_data_size ); + mf->set_addr( e->second.m_addr ); if ( m_config.m_alloc_policy == ON_MISS ) - m_tag_array->fill(e->second.m_cache_index,time); - else if ( m_config.m_alloc_policy == ON_FILL ) - m_tag_array->fill(e->second.m_block_addr,time); + m_tag_array->fill(e->second.m_cache_index,time,mf); + else if ( m_config.m_alloc_policy == ON_FILL ) { + m_tag_array->fill(e->second.m_block_addr,time,mf); + if(m_config.is_streaming()) + m_tag_array->remove_pending_line(mf); + } else abort(); bool has_atomic = false; m_mshrs.mark_ready(e->second.m_block_addr, has_atomic); if (has_atomic) { assert(m_config.m_alloc_policy == ON_MISS); - cache_block_t &block = m_tag_array->get_block(e->second.m_cache_index); - block.m_status = MODIFIED; // mark line as dirty for atomic operation + cache_block_t* block = m_tag_array->get_block(e->second.m_cache_index); + block->set_status(MODIFIED, mf->get_access_sector_mask()); // mark line as dirty for atomic operation } m_extra_mf_fields.erase(mf); m_bandwidth_management.use_fill_port(mf); @@ -749,45 +1017,59 @@ void baseline_cache::send_read_request(new_addr_type addr, new_addr_type block_a unsigned time, bool &do_miss, std::list &events, bool read_only, bool wa){ bool wb=false; - cache_block_t e; + evicted_block_info e; send_read_request(addr, block_addr, cache_index, mf, time, do_miss, wb, e, events, read_only, wa); } /// Read miss handler. Check MSHR hit or MSHR available void baseline_cache::send_read_request(new_addr_type addr, new_addr_type block_addr, unsigned cache_index, mem_fetch *mf, - unsigned time, bool &do_miss, bool &wb, cache_block_t &evicted, std::list &events, bool read_only, bool wa){ + unsigned time, bool &do_miss, bool &wb, evicted_block_info &evicted, std::list &events, bool read_only, bool wa){ - bool mshr_hit = m_mshrs.probe(block_addr); - bool mshr_avail = !m_mshrs.full(block_addr); + new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr()); + bool mshr_hit = m_mshrs.probe(mshr_addr); + bool mshr_avail = !m_mshrs.full(mshr_addr); if ( mshr_hit && mshr_avail ) { if(read_only) - m_tag_array->access(block_addr,time,cache_index); + m_tag_array->access(block_addr,time,cache_index,mf); else - m_tag_array->access(block_addr,time,cache_index,wb,evicted); + m_tag_array->access(block_addr,time,cache_index,wb,evicted,mf); - m_mshrs.add(block_addr,mf); + m_mshrs.add(mshr_addr,mf); do_miss = true; + } else if ( !mshr_hit && mshr_avail && (m_miss_queue.size() < m_config.m_miss_queue_size) ) { if(read_only) - m_tag_array->access(block_addr,time,cache_index); + m_tag_array->access(block_addr,time,cache_index,mf); else - m_tag_array->access(block_addr,time,cache_index,wb,evicted); - - m_mshrs.add(block_addr,mf); - m_extra_mf_fields[mf] = extra_mf_fields(block_addr,cache_index, mf->get_data_size()); - mf->set_data_size( m_config.get_line_sz() ); + m_tag_array->access(block_addr,time,cache_index,wb,evicted,mf); + + m_mshrs.add(mshr_addr,mf); + if(m_config.is_streaming() && m_config.m_cache_type == SECTOR){ + m_tag_array->add_pending_line(mf); + } + m_extra_mf_fields[mf] = extra_mf_fields(mshr_addr,mf->get_addr(),cache_index, mf->get_data_size(), m_config); + mf->set_data_size( m_config.get_atom_sz() ); + mf->set_addr( mshr_addr ); m_miss_queue.push_back(mf); mf->set_status(m_miss_queue_status,time); if(!wa) - events.push_back(READ_REQUEST_SENT); + events.push_back(cache_event(READ_REQUEST_SENT)); + do_miss = true; } + else if(mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_MERGE_ENRTY_FAIL); + else if (!mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_ENRTY_FAIL); + else + assert(0); } /// Sends write request to lower level memory (write or writeback) void data_cache::send_write_request(mem_fetch *mf, cache_event request, unsigned time, std::list &events){ - events.push_back(request); + + events.push_back(request); m_miss_queue.push_back(mf); mf->set_status(m_miss_queue_status,time); } @@ -798,40 +1080,44 @@ void data_cache::send_write_request(mem_fetch *mf, cache_event request, unsigned /// Write-back hit: Mark block as modified cache_request_status data_cache::wr_hit_wb(new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, std::list &events, enum cache_request_status status ){ new_addr_type block_addr = m_config.block_addr(addr); - m_tag_array->access(block_addr,time,cache_index); // update LRU state - cache_block_t &block = m_tag_array->get_block(cache_index); - block.m_status = MODIFIED; + m_tag_array->access(block_addr,time,cache_index,mf); // update LRU state + cache_block_t* block = m_tag_array->get_block(cache_index); + block->set_status(MODIFIED, mf->get_access_sector_mask()); return HIT; } /// Write-through hit: Directly send request to lower level memory cache_request_status data_cache::wr_hit_wt(new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, std::list &events, enum cache_request_status status ){ - if(miss_queue_full(0)) + if(miss_queue_full(0)) { + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); return RESERVATION_FAIL; // cannot handle request this cycle + } new_addr_type block_addr = m_config.block_addr(addr); - m_tag_array->access(block_addr,time,cache_index); // update LRU state - cache_block_t &block = m_tag_array->get_block(cache_index); - block.m_status = MODIFIED; + m_tag_array->access(block_addr,time,cache_index,mf); // update LRU state + cache_block_t* block = m_tag_array->get_block(cache_index); + block->set_status(MODIFIED, mf->get_access_sector_mask()); // generate a write-through - send_write_request(mf, WRITE_REQUEST_SENT, time, events); + send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events); return HIT; } /// Write-evict hit: Send request to lower level memory and invalidate corresponding block cache_request_status data_cache::wr_hit_we(new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, std::list &events, enum cache_request_status status ){ - if(miss_queue_full(0)) + if(miss_queue_full(0)) { + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); return RESERVATION_FAIL; // cannot handle request this cycle + } // generate a write-through/evict - cache_block_t &block = m_tag_array->get_block(cache_index); - send_write_request(mf, WRITE_REQUEST_SENT, time, events); + cache_block_t* block = m_tag_array->get_block(cache_index); + send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events); // Invalidate block - block.m_status = INVALID; + block->set_status(INVALID, mf->get_access_sector_mask()); return HIT; } @@ -850,34 +1136,46 @@ enum cache_request_status data_cache::wr_hit_global_we_local_wb(new_addr_type ad /// Write-allocate miss: Send write request to lower level memory // and send a read request for the same block enum cache_request_status -data_cache::wr_miss_wa( new_addr_type addr, +data_cache::wr_miss_wa_naive( new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, std::list &events, enum cache_request_status status ) { new_addr_type block_addr = m_config.block_addr(addr); + new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr()); // Write allocate, maximum 3 requests (write miss, read request, write back request) // Conservatively ensure the worst-case request can be handled this cycle - bool mshr_hit = m_mshrs.probe(block_addr); - bool mshr_avail = !m_mshrs.full(block_addr); + bool mshr_hit = m_mshrs.probe(mshr_addr); + bool mshr_avail = !m_mshrs.full(mshr_addr); if(miss_queue_full(2) || (!(mshr_hit && mshr_avail) - && !(!mshr_hit && mshr_avail - && (m_miss_queue.size() < m_config.m_miss_queue_size)))) + && !(!mshr_hit && mshr_avail && (m_miss_queue.size() < m_config.m_miss_queue_size)))) { + //check what is the exactly the failure reason + if(miss_queue_full(2) ) + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + else if(mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_MERGE_ENRTY_FAIL); + else if (!mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_ENRTY_FAIL); + else + assert(0); + return RESERVATION_FAIL; + } - send_write_request(mf, WRITE_REQUEST_SENT, time, events); + send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events); // Tries to send write allocate request, returns true on success and false on failure //if(!send_write_allocate(mf, addr, block_addr, cache_index, time, events)) // return RESERVATION_FAIL; const mem_access_t *ma = new mem_access_t( m_wr_alloc_type, mf->get_addr(), - mf->get_data_size(), + m_config.get_atom_sz(), false, // Now performing a read mf->get_access_warp_mask(), - mf->get_access_byte_mask() ); + mf->get_access_byte_mask(), + mf->get_access_sector_mask()); mem_fetch *n_mf = new mem_fetch( *ma, NULL, @@ -889,20 +1187,22 @@ data_cache::wr_miss_wa( new_addr_type addr, bool do_miss = false; bool wb = false; - cache_block_t evicted; + evicted_block_info evicted; // Send read request resulting from write miss send_read_request(addr, block_addr, cache_index, n_mf, time, do_miss, wb, evicted, events, false, true); + events.push_back(cache_event(WRITE_ALLOCATE_SENT)); + if( do_miss ){ // If evicted block is modified and not a write-through // (already modified lower level) if( wb && (m_config.m_write_policy != WRITE_THROUGH) ) { + assert(status == MISS); //SECTOR_MISS and HIT_RESERVED should not send write back mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr, - m_wrbk_type,m_config.get_line_sz(),true); - m_miss_queue.push_back(wb); - wb->set_status(m_miss_queue_status,time); + m_wrbk_type,evicted.m_modified_size,true); + send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), time, events); } return MISS; } @@ -910,6 +1210,177 @@ data_cache::wr_miss_wa( new_addr_type addr, return RESERVATION_FAIL; } + +enum cache_request_status +data_cache::wr_miss_wa_fetch_on_write( new_addr_type addr, + unsigned cache_index, mem_fetch *mf, + unsigned time, std::list &events, + enum cache_request_status status ) +{ + new_addr_type block_addr = m_config.block_addr(addr); + new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr()); + + if(mf->get_access_byte_mask().count() == m_config.get_atom_sz()) + { + //if the request writes to the whole cache line/sector, then, write and set cache line Modified. + //and no need to send read request to memory or reserve mshr + + if(miss_queue_full(0)) { + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + return RESERVATION_FAIL; // cannot handle request this cycle + } + + bool wb = false; + evicted_block_info evicted; + + cache_request_status status = m_tag_array->access(block_addr,time,cache_index,wb,evicted,mf); + assert(status != HIT); + cache_block_t* block = m_tag_array->get_block(cache_index); + block->set_status(MODIFIED, mf->get_access_sector_mask()); + if(status == HIT_RESERVED) + block->set_ignore_on_fill(true, mf->get_access_sector_mask()); + + if( status != RESERVATION_FAIL ){ + // If evicted block is modified and not a write-through + // (already modified lower level) + if( wb && (m_config.m_write_policy != WRITE_THROUGH) ) { + mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr, + m_wrbk_type,evicted.m_modified_size,true); + send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), time, events); + } + return MISS; + } + return RESERVATION_FAIL; + } + else + { + bool mshr_hit = m_mshrs.probe(mshr_addr); + bool mshr_avail = !m_mshrs.full(mshr_addr); + if(miss_queue_full(1) + || (!(mshr_hit && mshr_avail) + && !(!mshr_hit && mshr_avail && (m_miss_queue.size() < m_config.m_miss_queue_size)))) { + //check what is the exactly the failure reason + if(miss_queue_full(1) ) + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + else if(mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_MERGE_ENRTY_FAIL); + else if (!mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_ENRTY_FAIL); + else + assert(0); + + return RESERVATION_FAIL; + } + + + //prevent Write - Read - Write in pending mshr + //allowing another write will override the value of the first write, and the pending read request will read incorrect result from the second write + if(m_mshrs.probe(mshr_addr) && m_mshrs.is_read_after_write_pending(mshr_addr) && mf->is_write()) + { + //assert(0); + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_RW_PENDING); + return RESERVATION_FAIL; + } + + const mem_access_t *ma = new mem_access_t( m_wr_alloc_type, + mf->get_addr(), + m_config.get_atom_sz(), + false, // Now performing a read + mf->get_access_warp_mask(), + mf->get_access_byte_mask(), + mf->get_access_sector_mask()); + + mem_fetch *n_mf = new mem_fetch( *ma, + NULL, + mf->get_ctrl_size(), + mf->get_wid(), + mf->get_sid(), + mf->get_tpc(), + mf->get_mem_config(), + NULL, + mf); + + + new_addr_type block_addr = m_config.block_addr(addr); + bool do_miss = false; + bool wb = false; + evicted_block_info evicted; + send_read_request( addr, + block_addr, + cache_index, + n_mf, time, do_miss, wb, evicted, events, false, true); + + cache_block_t* block = m_tag_array->get_block(cache_index); + block->set_modified_on_fill(true, mf->get_access_sector_mask()); + + events.push_back(cache_event(WRITE_ALLOCATE_SENT)); + + if( do_miss ){ + // If evicted block is modified and not a write-through + // (already modified lower level) + if(wb && (m_config.m_write_policy != WRITE_THROUGH) ){ + mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr, + m_wrbk_type,evicted.m_modified_size,true); + send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), time, events); + } + return MISS; + } + return RESERVATION_FAIL; + } +} + +enum cache_request_status +data_cache::wr_miss_wa_lazy_fetch_on_read( new_addr_type addr, + unsigned cache_index, mem_fetch *mf, + unsigned time, std::list &events, + enum cache_request_status status ) +{ + + new_addr_type block_addr = m_config.block_addr(addr); + new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr()); + + + //if the request writes to the whole cache line/sector, then, write and set cache line Modified. + //and no need to send read request to memory or reserve mshr + + if(miss_queue_full(0)) { + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + return RESERVATION_FAIL; // cannot handle request this cycle + } + + bool wb = false; + evicted_block_info evicted; + + cache_request_status m_status = m_tag_array->access(block_addr,time,cache_index,wb,evicted,mf); + assert(m_status != HIT); + cache_block_t* block = m_tag_array->get_block(cache_index); + block->set_status(MODIFIED, mf->get_access_sector_mask()); + if(m_status == HIT_RESERVED) { + block->set_ignore_on_fill(true, mf->get_access_sector_mask()); + block->set_modified_on_fill(true, mf->get_access_sector_mask()); + } + + if(mf->get_access_byte_mask().count() == m_config.get_atom_sz()) + { + block->set_m_readable(true, mf->get_access_sector_mask()); + } else + { + block->set_m_readable(false, mf->get_access_sector_mask()); + } + + if( m_status != RESERVATION_FAIL ){ + // If evicted block is modified and not a write-through + // (already modified lower level) + if( wb && (m_config.m_write_policy != WRITE_THROUGH) ) { + mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr, + m_wrbk_type,evicted.m_modified_size,true); + send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), time, events); + } + return MISS; + } + return RESERVATION_FAIL; +} + /// No write-allocate miss: Simply send write request to lower level memory enum cache_request_status data_cache::wr_miss_no_wa( new_addr_type addr, @@ -919,11 +1390,14 @@ data_cache::wr_miss_no_wa( new_addr_type addr, std::list &events, enum cache_request_status status ) { - if(miss_queue_full(0)) - return RESERVATION_FAIL; // cannot handle request this cycle + if(miss_queue_full(0)) { + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + return RESERVATION_FAIL; // cannot handle request this cycle + } + // on miss, generate write through (no write buffering -- too many threads for that) - send_write_request(mf, WRITE_REQUEST_SENT, time, events); + send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events); return MISS; } @@ -941,13 +1415,13 @@ data_cache::rd_hit_base( new_addr_type addr, enum cache_request_status status ) { new_addr_type block_addr = m_config.block_addr(addr); - m_tag_array->access(block_addr,time,cache_index); + m_tag_array->access(block_addr,time,cache_index,mf); // Atomics treated as global read/write requests - Perform read, mark line as // MODIFIED if(mf->isatomic()){ assert(mf->get_access_type() == GLOBAL_ACC_R); - cache_block_t &block = m_tag_array->get_block(cache_index); - block.m_status = MODIFIED; // mark line as dirty + cache_block_t* block = m_tag_array->get_block(cache_index); + block->set_status(MODIFIED, mf->get_access_sector_mask()) ; // mark line as dirty } return HIT; } @@ -963,15 +1437,17 @@ data_cache::rd_miss_base( new_addr_type addr, unsigned time, std::list &events, enum cache_request_status status ){ - if(miss_queue_full(1)) + if(miss_queue_full(1)) { // cannot handle request this cycle // (might need to generate two requests) + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); return RESERVATION_FAIL; + } new_addr_type block_addr = m_config.block_addr(addr); bool do_miss = false; bool wb = false; - cache_block_t evicted; + evicted_block_info evicted; send_read_request( addr, block_addr, cache_index, @@ -982,12 +1458,12 @@ data_cache::rd_miss_base( new_addr_type addr, // (already modified lower level) if(wb && (m_config.m_write_policy != WRITE_THROUGH) ){ mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr, - m_wrbk_type,m_config.get_line_sz(),true); + m_wrbk_type,evicted.m_modified_size,true); send_write_request(wb, WRITE_BACK_REQUEST_SENT, time, events); } return MISS; } - return RESERVATION_FAIL; + return RESERVATION_FAIL; } /// Access cache for read_only_cache: returns RESERVATION_FAIL if @@ -998,16 +1474,16 @@ read_only_cache::access( new_addr_type addr, unsigned time, std::list &events ) { - assert( mf->get_data_size() <= m_config.get_line_sz()); + assert( mf->get_data_size() <= m_config.get_atom_sz()); assert(m_config.m_write_policy == READ_ONLY); assert(!mf->get_is_write()); new_addr_type block_addr = m_config.block_addr(addr); unsigned cache_index = (unsigned)-1; - enum cache_request_status status = m_tag_array->probe(block_addr,cache_index); + enum cache_request_status status = m_tag_array->probe(block_addr,cache_index,mf); enum cache_request_status cache_status = RESERVATION_FAIL; if ( status == HIT ) { - cache_status = m_tag_array->access(block_addr,time,cache_index); // update LRU state + cache_status = m_tag_array->access(block_addr,time,cache_index,mf); // update LRU state }else if ( status != RESERVATION_FAIL ) { if(!miss_queue_full(0)){ bool do_miss=false; @@ -1018,7 +1494,10 @@ read_only_cache::access( new_addr_type addr, cache_status = RESERVATION_FAIL; }else{ cache_status = RESERVATION_FAIL; + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); } + }else { + m_stats.inc_fail_stats(mf->get_access_type(), LINE_ALLOC_FAIL); } m_stats.inc_stats(mf->get_access_type(), m_stats.select_stats_status(status, cache_status)); @@ -1047,10 +1526,13 @@ data_cache::process_tag_probe( bool wr, access_status = (this->*m_wr_hit)( addr, cache_index, mf, time, events, probe_status ); - }else if ( probe_status != RESERVATION_FAIL ) { + }else if ( (probe_status != RESERVATION_FAIL) || (probe_status == RESERVATION_FAIL && m_config.m_write_alloc_policy == NO_WRITE_ALLOCATE) ) { access_status = (this->*m_wr_miss)( addr, cache_index, mf, time, events, probe_status ); + }else { + //the only reason for reservation fail here is LINE_ALLOC_FAIL (i.e all lines are reserved) + m_stats.inc_fail_stats(mf->get_access_type(), LINE_ALLOC_FAIL); } }else{ // Read if(probe_status == HIT){ @@ -1061,6 +1543,9 @@ data_cache::process_tag_probe( bool wr, access_status = (this->*m_rd_miss)( addr, cache_index, mf, time, events, probe_status ); + }else { + //the only reason for reservation fail here is LINE_ALLOC_FAIL (i.e all lines are reserved) + m_stats.inc_fail_stats(mf->get_access_type(), LINE_ALLOC_FAIL); } } @@ -1080,12 +1565,12 @@ data_cache::access( new_addr_type addr, std::list &events ) { - assert( mf->get_data_size() <= m_config.get_line_sz()); + assert( mf->get_data_size() <= m_config.get_atom_sz()); bool wr = mf->get_is_write(); new_addr_type block_addr = m_config.block_addr(addr); unsigned cache_index = (unsigned)-1; enum cache_request_status probe_status - = m_tag_array->probe( block_addr, cache_index ); + = m_tag_array->probe( block_addr, cache_index, mf, true); enum cache_request_status access_status = process_tag_probe( wr, probe_status, addr, cache_index, mf, time, events ); m_stats.inc_stats(mf->get_access_type(), @@ -1134,7 +1619,7 @@ enum cache_request_status tex_cache::access( new_addr_type addr, mem_fetch *mf, // at this point, we will accept the request : access tags and immediately allocate line new_addr_type block_addr = m_config.block_addr(addr); unsigned cache_index = (unsigned)-1; - enum cache_request_status status = m_tags.access(block_addr,time,cache_index); + enum cache_request_status status = m_tags.access(block_addr,time,cache_index,mf); enum cache_request_status cache_status = RESERVATION_FAIL; assert( status != RESERVATION_FAIL ); assert( status != HIT_RESERVED ); // as far as tags are concerned: HIT or MISS @@ -1142,12 +1627,12 @@ enum cache_request_status tex_cache::access( new_addr_type addr, mem_fetch *mf, if ( status == MISS ) { // we need to send a memory request... unsigned rob_index = m_rob.push( rob_entry(cache_index, mf, block_addr) ); - m_extra_mf_fields[mf] = extra_mf_fields(rob_index); + m_extra_mf_fields[mf] = extra_mf_fields(rob_index, m_config); mf->set_data_size(m_config.get_line_sz()); - m_tags.fill(cache_index,time); // mark block as valid + m_tags.fill(cache_index,time,mf); // mark block as valid m_request_fifo.push(mf); mf->set_status(m_request_queue_status,time); - events.push_back(READ_REQUEST_SENT); + events.push_back(cache_event(READ_REQUEST_SENT)); cache_status = MISS; } else { // the value *will* *be* in the cache already @@ -1174,7 +1659,7 @@ void tex_cache::cycle(){ unsigned rob_index = m_rob.next_pop_index(); const rob_entry &r = m_rob.peek(rob_index); assert( r.m_request == e.m_request ); - assert( r.m_block_addr == m_config.block_addr(e.m_request->get_addr()) ); + //assert( r.m_block_addr == m_config.block_addr(e.m_request->get_addr()) ); if ( r.m_ready ) { assert( r.m_index == e.m_cache_index ); m_cache[r.m_index].m_valid = true; @@ -1197,6 +1682,23 @@ void tex_cache::cycle(){ /// Place returning cache block into reorder buffer void tex_cache::fill( mem_fetch *mf, unsigned time ) { + if(m_config.m_mshr_type == SECTOR_TEX_FIFO) { + assert(mf->get_original_mf()); + extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf->get_original_mf()); + assert( e != m_extra_mf_fields.end() ); + e->second.pending_read--; + + if(e->second.pending_read > 0) { + //wait for the other requests to come back + delete mf; + return; + } else { + mem_fetch *temp = mf; + mf = mf->get_original_mf(); + delete temp; + } + } + extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf); assert( e != m_extra_mf_fields.end() ); assert( e->second.m_valid ); diff --git a/src/gpgpu-sim/gpu-cache.h b/src/gpgpu-sim/gpu-cache.h index 7535a1d..e663cf6 100644 --- a/src/gpgpu-sim/gpu-cache.h +++ b/src/gpgpu-sim/gpu-cache.h @@ -36,9 +36,12 @@ #include "../tr1_hash_map.h" #include "addrdec.h" +#include + +#define MAX_DEFAULT_CACHE_SIZE_MULTIBLIER 4 enum cache_block_state { - INVALID, + INVALID=0, RESERVED, VALID, MODIFIED @@ -49,13 +52,51 @@ enum cache_request_status { HIT_RESERVED, MISS, RESERVATION_FAIL, + SECTOR_MISS, NUM_CACHE_REQUEST_STATUS }; -enum cache_event { +enum cache_reservation_fail_reason { + LINE_ALLOC_FAIL= 0,// all line are reserved + MISS_QUEUE_FULL, // MISS queue (i.e. interconnect or DRAM) is full + MSHR_ENRTY_FAIL, + MSHR_MERGE_ENRTY_FAIL, + MSHR_RW_PENDING, + NUM_CACHE_RESERVATION_FAIL_STATUS +}; + +enum cache_event_type { WRITE_BACK_REQUEST_SENT, READ_REQUEST_SENT, - WRITE_REQUEST_SENT + WRITE_REQUEST_SENT, + WRITE_ALLOCATE_SENT +}; + +struct evicted_block_info { + new_addr_type m_block_addr; + unsigned m_modified_size; + evicted_block_info() { + m_block_addr = 0; + m_modified_size = 0; + } + void set_info(new_addr_type block_addr, unsigned modified_size){ + m_block_addr = block_addr; + m_modified_size = modified_size; + } +}; + +struct cache_event { + enum cache_event_type m_cache_event_type; + evicted_block_info m_evicted_block; //if it was write_back event, fill the the evicted block info + + cache_event(enum cache_event_type m_cache_event){ + m_cache_event_type = m_cache_event; + } + + cache_event(enum cache_event_type cache_event, evicted_block_info evicted_block){ + m_cache_event_type = cache_event; + m_evicted_block = evicted_block; + } }; const char * cache_request_status_str(enum cache_request_status status); @@ -65,33 +106,340 @@ struct cache_block_t { { m_tag=0; m_block_addr=0; - m_alloc_time=0; - m_fill_time=0; - m_last_access_time=0; - m_status=INVALID; } - void allocate( new_addr_type tag, new_addr_type block_addr, unsigned time ) + + virtual void allocate( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask) = 0; + virtual void fill( unsigned time, mem_access_sector_mask_t sector_mask) = 0; + + virtual bool is_invalid_line() = 0; + virtual bool is_valid_line() = 0; + virtual bool is_reserved_line() = 0; + virtual bool is_modified_line() = 0; + + virtual enum cache_block_state get_status( mem_access_sector_mask_t sector_mask) = 0; + virtual void set_status(enum cache_block_state m_status, mem_access_sector_mask_t sector_mask) = 0; + + virtual unsigned get_last_access_time() = 0; + virtual void set_last_access_time(unsigned time, mem_access_sector_mask_t sector_mask) = 0; + virtual unsigned get_alloc_time() = 0; + virtual void set_ignore_on_fill(bool m_ignore, mem_access_sector_mask_t sector_mask) = 0; + virtual void set_modified_on_fill(bool m_modified, mem_access_sector_mask_t sector_mask) = 0; + virtual unsigned get_modified_size() = 0; + virtual void set_m_readable(bool readable, mem_access_sector_mask_t sector_mask)=0; + virtual bool is_readable(mem_access_sector_mask_t sector_mask)=0; + virtual void print_status()=0; + virtual ~cache_block_t() {} + + + new_addr_type m_tag; + new_addr_type m_block_addr; + +}; + +struct line_cache_block: public cache_block_t { + line_cache_block() + { + m_alloc_time=0; + m_fill_time=0; + m_last_access_time=0; + m_status=INVALID; + m_ignore_on_fill_status = false; + m_set_modified_on_fill = false; + m_readable = true; + } + void allocate( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask) + { + m_tag=tag; + m_block_addr=block_addr; + m_alloc_time=time; + m_last_access_time=time; + m_fill_time=0; + m_status=RESERVED; + m_ignore_on_fill_status = false; + m_set_modified_on_fill = false; + } + void fill( unsigned time, mem_access_sector_mask_t sector_mask ) + { + //if(!m_ignore_on_fill_status) + // assert( m_status == RESERVED ); + + m_status = m_set_modified_on_fill? MODIFIED : VALID; + + m_fill_time=time; + } + virtual bool is_invalid_line() + { + return m_status == INVALID; + } + virtual bool is_valid_line() + { + return m_status == VALID; + } + virtual bool is_reserved_line() + { + return m_status == RESERVED; + } + virtual bool is_modified_line() + { + return m_status == MODIFIED; + } + + virtual enum cache_block_state get_status(mem_access_sector_mask_t sector_mask) + { + return m_status; + } + virtual void set_status(enum cache_block_state status, mem_access_sector_mask_t sector_mask) + { + m_status = status; + } + virtual unsigned get_last_access_time() + { + return m_last_access_time; + } + virtual void set_last_access_time(unsigned time, mem_access_sector_mask_t sector_mask) + { + m_last_access_time = time; + } + virtual unsigned get_alloc_time() + { + return m_alloc_time; + } + virtual void set_ignore_on_fill(bool m_ignore, mem_access_sector_mask_t sector_mask) + { + m_ignore_on_fill_status = m_ignore; + } + virtual void set_modified_on_fill(bool m_modified, mem_access_sector_mask_t sector_mask) + { + m_set_modified_on_fill = m_modified; + } + virtual unsigned get_modified_size() + { + return SECTOR_CHUNCK_SIZE * SECTOR_SIZE; //i.e. cache line size + } + virtual void set_m_readable(bool readable, mem_access_sector_mask_t sector_mask) + { + m_readable = readable; + } + virtual bool is_readable(mem_access_sector_mask_t sector_mask) { + return m_readable; + } + virtual void print_status() { + printf("m_block_addr is %llu, status = %u\n", m_block_addr, m_status); + } + + +private: + unsigned m_alloc_time; + unsigned m_last_access_time; + unsigned m_fill_time; + cache_block_state m_status; + bool m_ignore_on_fill_status; + bool m_set_modified_on_fill; + bool m_readable; +}; + +struct sector_cache_block : public cache_block_t { + sector_cache_block() { - m_tag=tag; - m_block_addr=block_addr; - m_alloc_time=time; - m_last_access_time=time; - m_fill_time=0; - m_status=RESERVED; + init(); } - void fill( unsigned time ) + + void init() { + for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { + m_sector_alloc_time[i]= 0; + m_sector_fill_time[i]= 0; + m_last_sector_access_time[i]= 0; + m_status[i]= INVALID; + m_ignore_on_fill_status[i] = false; + m_set_modified_on_fill[i] = false; + m_readable[i] = true; + } + m_line_alloc_time=0; + m_line_last_access_time=0; + m_line_fill_time=0; + } + + virtual void allocate( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask ) { - assert( m_status == RESERVED ); - m_status=VALID; - m_fill_time=time; + allocate_line( tag, block_addr, time, sector_mask ); } - new_addr_type m_tag; - new_addr_type m_block_addr; - unsigned m_alloc_time; - unsigned m_last_access_time; - unsigned m_fill_time; - cache_block_state m_status; + void allocate_line( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask ) + { + //allocate a new line + //assert(m_block_addr != 0 && m_block_addr != block_addr); + init(); + m_tag=tag; + m_block_addr=block_addr; + + unsigned sidx = get_sector_index(sector_mask); + + //set sector stats + m_sector_alloc_time[sidx]=time; + m_last_sector_access_time[sidx]=time; + m_sector_fill_time[sidx]=0; + m_status[sidx]=RESERVED; + m_ignore_on_fill_status[sidx] = false; + m_set_modified_on_fill[sidx] = false; + + //set line stats + m_line_alloc_time=time; //only set this for the first allocated sector + m_line_last_access_time=time; + m_line_fill_time=0; + } + + void allocate_sector(unsigned time, mem_access_sector_mask_t sector_mask ) + { + //allocate invalid sector of this allocated valid line + assert(is_valid_line()); + unsigned sidx = get_sector_index(sector_mask); + + //set sector stats + m_sector_alloc_time[sidx]=time; + m_last_sector_access_time[sidx]=time; + m_sector_fill_time[sidx]=0; + if(m_status[sidx]==MODIFIED) //this should be the case only for fetch-on-write policy //TO DO + m_set_modified_on_fill[sidx] = true; + else + m_set_modified_on_fill[sidx] = false; + + m_status[sidx]=RESERVED; + m_ignore_on_fill_status[sidx] = false; + //m_set_modified_on_fill[sidx] = false; + m_readable[sidx] = true; + + //set line stats + m_line_last_access_time=time; + m_line_fill_time=0; + } + + virtual void fill( unsigned time, mem_access_sector_mask_t sector_mask) + { + unsigned sidx = get_sector_index(sector_mask); + + // if(!m_ignore_on_fill_status[sidx]) + // assert( m_status[sidx] == RESERVED ); + + m_status[sidx] = m_set_modified_on_fill[sidx]? MODIFIED : VALID; + + m_sector_fill_time[sidx]=time; + m_line_fill_time=time; + } + virtual bool is_invalid_line() { + //all the sectors should be invalid + for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { + if (m_status[i] != INVALID) + return false; + } + return true; + } + virtual bool is_valid_line() { return !(is_invalid_line()); } + virtual bool is_reserved_line() { + //if any of the sector is reserved, then the line is reserved + for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { + if (m_status[i] == RESERVED) + return true; + } + return false; + } + virtual bool is_modified_line() { + //if any of the sector is modified, then the line is modified + for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { + if (m_status[i] == MODIFIED) + return true; + } + return false; + } + + virtual enum cache_block_state get_status(mem_access_sector_mask_t sector_mask) + { + unsigned sidx = get_sector_index(sector_mask); + + return m_status[sidx]; + } + + virtual void set_status(enum cache_block_state status, mem_access_sector_mask_t sector_mask) + { + unsigned sidx = get_sector_index(sector_mask); + m_status[sidx] = status; + } + + virtual unsigned get_last_access_time() + { + return m_line_last_access_time; + } + + virtual void set_last_access_time(unsigned time, mem_access_sector_mask_t sector_mask) + { + unsigned sidx = get_sector_index(sector_mask); + + m_last_sector_access_time[sidx] = time; + m_line_last_access_time = time; + } + + virtual unsigned get_alloc_time() + { + return m_line_alloc_time; + } + + virtual void set_ignore_on_fill(bool m_ignore, mem_access_sector_mask_t sector_mask) + { + unsigned sidx = get_sector_index(sector_mask); + m_ignore_on_fill_status[sidx] = m_ignore; + } + + virtual void set_modified_on_fill(bool m_modified, mem_access_sector_mask_t sector_mask) + { + unsigned sidx = get_sector_index(sector_mask); + m_set_modified_on_fill[sidx] = m_modified; + } + + virtual void set_m_readable(bool readable, mem_access_sector_mask_t sector_mask) + { + unsigned sidx = get_sector_index(sector_mask); + m_readable[sidx] = readable; + } + + virtual bool is_readable(mem_access_sector_mask_t sector_mask) { + unsigned sidx = get_sector_index(sector_mask); + return m_readable[sidx]; + } + + virtual unsigned get_modified_size() + { + unsigned modified=0; + for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { + if (m_status[i] == MODIFIED) + modified++; + } + return modified * SECTOR_SIZE; + } + + virtual void print_status() { + printf("m_block_addr is %llu, status = %u %u %u %u\n", m_block_addr, m_status[0], m_status[1], m_status[2], m_status[3]); + } + + +private: + unsigned m_sector_alloc_time[SECTOR_CHUNCK_SIZE]; + unsigned m_last_sector_access_time[SECTOR_CHUNCK_SIZE]; + unsigned m_sector_fill_time[SECTOR_CHUNCK_SIZE]; + unsigned m_line_alloc_time; + unsigned m_line_last_access_time; + unsigned m_line_fill_time; + cache_block_state m_status[SECTOR_CHUNCK_SIZE]; + bool m_ignore_on_fill_status[SECTOR_CHUNCK_SIZE]; + bool m_set_modified_on_fill[SECTOR_CHUNCK_SIZE]; + bool m_readable[SECTOR_CHUNCK_SIZE]; + + unsigned get_sector_index(mem_access_sector_mask_t sector_mask) + { + assert(sector_mask.count() == 1); + for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { + if(sector_mask.to_ulong() & (1< line_table; + line_table pending_lines; }; class mshr_table { public: - mshr_table( unsigned num_entries, unsigned max_merged ) + mshr_table( unsigned num_entries, unsigned max_merged) : m_num_entries(num_entries), m_max_merged(max_merged) #if (tr1_hash_map_ismap == 0) @@ -414,6 +879,8 @@ public: /// Returns next ready access mem_fetch *next_access(); void display( FILE *fp ) const; + // Returns true if there is a pending read after write + bool is_read_after_write_pending(new_addr_type block_addr); void check_mshr_parameters( unsigned num_entries, unsigned max_merged ) { @@ -433,7 +900,9 @@ private: mshr_entry() : m_has_atomic(false) { } }; typedef tr1_hash_map table; + typedef tr1_hash_map line_table; table m_data; + line_table pending_lines; // it may take several cycles to process the merged requests bool m_current_response_ready; @@ -510,12 +979,14 @@ public: cache_stats(); void clear(); void inc_stats(int access_type, int access_outcome); + void inc_fail_stats(int access_type, int fail_outcome); enum cache_request_status select_stats_status(enum cache_request_status probe, enum cache_request_status access) const; - unsigned &operator()(int access_type, int access_outcome); - unsigned operator()(int access_type, int access_outcome) const; + unsigned &operator()(int access_type, int access_outcome, bool fail_outcome); + unsigned operator()(int access_type, int access_outcome, bool fail_outcome) const; cache_stats operator+(const cache_stats &cs); cache_stats &operator+=(const cache_stats &cs); void print_stats(FILE *fout, const char *cache_name = "Cache_stats") const; + void print_fail_stats(FILE *fout, const char *cache_name = "Cache_fail_stats") const; unsigned get_stats(enum mem_access_type *access_type, unsigned num_access_type, enum cache_request_status *access_status, unsigned num_access_status) const; void get_sub_stats(struct cache_sub_stats &css) const; @@ -523,8 +994,10 @@ public: void sample_cache_port_utility(bool data_port_busy, bool fill_port_busy); private: bool check_valid(int type, int status) const; + bool check_fail_valid(int type, int fail) const; std::vector< std::vector > m_stats; + std::vector< std::vector > m_fail_stats; unsigned long long m_cache_port_available_cycles; unsigned long long m_cache_data_port_busy_cycles; @@ -543,6 +1016,7 @@ public: bool was_write_sent( const std::list &events ); bool was_read_sent( const std::list &events ); +bool was_writeallocate_sent( const std::list &events ); /// Baseline cache /// Implements common functions for read_only_cache and data_cache @@ -552,7 +1026,7 @@ public: baseline_cache( const char *name, cache_config &config, int core_id, int type_id, mem_fetch_interface *memport, enum mem_fetch_status status ) : m_config(config), m_tag_array(new tag_array(config,core_id,type_id)), - m_mshrs(config.m_mshr_entries,config.m_mshr_max_merge), + m_mshrs(config.m_mshr_entries,config.m_mshr_max_merge), m_bandwidth_management(config) { init( name, config, memport, status ); @@ -564,7 +1038,7 @@ public: enum mem_fetch_status status ) { m_name = name; - assert(config.m_mshr_type == ASSOC); + assert(config.m_mshr_type == ASSOC || config.m_mshr_type == SECTOR_ASSOC); m_memport=memport; m_miss_queue_status = status; } @@ -594,6 +1068,7 @@ public: mem_fetch *next_access(){return m_mshrs.next_access();} // flash invalidate all entries in cache void flush(){m_tag_array->flush();} + void invalidate(){m_tag_array->invalidate();} void print(FILE *fp, unsigned &accesses, unsigned &misses) const; void display_state( FILE *fp ) const; @@ -612,6 +1087,15 @@ public: bool data_port_free() const { return m_bandwidth_management.data_port_free(); } bool fill_port_free() const { return m_bandwidth_management.fill_port_free(); } + // This is a gapping hole we are poking in the system to quickly handle + // filling the cache on cudamemcopies. We don't care about anything other than + // L2 state after the memcopy - so just force the tag array to act as though + // something is read or written without doing anything else. + void force_tag_access( new_addr_type addr, unsigned time, mem_access_sector_mask_t mask ) + { + m_tag_array->fill( addr, time, mask ); + } + protected: // Constructor that can be used by derived classes with custom tag arrays baseline_cache( const char *name, @@ -633,24 +1117,31 @@ protected: std::string m_name; cache_config &m_config; tag_array* m_tag_array; - mshr_table m_mshrs; + mshr_table m_mshrs; std::list m_miss_queue; enum mem_fetch_status m_miss_queue_status; mem_fetch_interface *m_memport; struct extra_mf_fields { extra_mf_fields() { m_valid = false;} - extra_mf_fields( new_addr_type a, unsigned i, unsigned d ) + extra_mf_fields( new_addr_type a, new_addr_type ad, unsigned i, unsigned d, const cache_config& m_config) { m_valid = true; m_block_addr = a; + m_addr = ad; m_cache_index = i; m_data_size = d; + pending_read = m_config.m_mshr_type == SECTOR_ASSOC? m_config.m_line_sz/SECTOR_SIZE : 0; + } bool m_valid; new_addr_type m_block_addr; + new_addr_type m_addr; unsigned m_cache_index; unsigned m_data_size; + //this variable is used when a load request generates multiple load transactions + //For example, a read request from non-sector L1 request sends a request to sector L2 + unsigned pending_read; }; typedef std::map extra_mf_fields_lookup; @@ -668,7 +1159,7 @@ protected: unsigned time, bool &do_miss, std::list &events, bool read_only, bool wa); /// Read miss handler. Check MSHR hit or MSHR available void send_read_request(new_addr_type addr, new_addr_type block_addr, unsigned cache_index, mem_fetch *mf, - unsigned time, bool &do_miss, bool &wb, cache_block_t &evicted, std::list &events, bool read_only, bool wa); + unsigned time, bool &do_miss, bool &wb, evicted_block_info &evicted, std::list &events, bool read_only, bool wa); /// Sub-class containing all metadata for port bandwidth management class bandwidth_management @@ -760,8 +1251,10 @@ public: // Set write miss function switch(m_config.m_write_alloc_policy){ - case WRITE_ALLOCATE: m_wr_miss = &data_cache::wr_miss_wa; break; case NO_WRITE_ALLOCATE: m_wr_miss = &data_cache::wr_miss_no_wa; break; + case WRITE_ALLOCATE: m_wr_miss = &data_cache::wr_miss_wa_naive; break; + case FETCH_ON_WRITE: m_wr_miss = &data_cache::wr_miss_wa_fetch_on_write; break; + case LAZY_FETCH_ON_READ: m_wr_miss = &data_cache::wr_miss_wa_lazy_fetch_on_read; break; default: assert(0 && "Error: Must set valid cache write miss policy\n"); break; // Need to set a write miss function @@ -870,12 +1363,33 @@ protected: /// Sends read request, and possible write-back request, // to lower level memory for a write miss with write-allocate enum cache_request_status - wr_miss_wa( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list &events, - enum cache_request_status status ); // write-allocate + wr_miss_wa_naive( new_addr_type addr, + unsigned cache_index, + mem_fetch *mf, + unsigned time, + std::list &events, + enum cache_request_status status ); // write-allocate-send-write-and-read-request + enum cache_request_status + wr_miss_wa_fetch_on_write( new_addr_type addr, + unsigned cache_index, + mem_fetch *mf, + unsigned time, + std::list &events, + enum cache_request_status status ); // write-allocate with fetch-on-every-write + enum cache_request_status + wr_miss_wa_lazy_fetch_on_read( new_addr_type addr, + unsigned cache_index, + mem_fetch *mf, + unsigned time, + std::list &events, + enum cache_request_status status ); // write-allocate with read-fetch-only + enum cache_request_status + wr_miss_wa_write_validate( new_addr_type addr, + unsigned cache_index, + mem_fetch *mf, + unsigned time, + std::list &events, + enum cache_request_status status ); // write-allocate that writes with no read fetch enum cache_request_status wr_miss_no_wa( new_addr_type addr, unsigned cache_index, @@ -991,7 +1505,7 @@ public: m_result_fifo(config.m_result_fifo_entries) { m_name = name; - assert(config.m_mshr_type == TEX_FIFO); + assert(config.m_mshr_type == TEX_FIFO || config.m_mshr_type == SECTOR_TEX_FIFO ); assert(config.m_write_policy == READ_ONLY); assert(config.m_alloc_policy == ON_MISS); m_memport=memport; @@ -1144,13 +1658,15 @@ private: struct extra_mf_fields { extra_mf_fields() { m_valid = false;} - extra_mf_fields( unsigned i ) + extra_mf_fields( unsigned i, const cache_config &m_config ) { m_valid = true; m_rob_index = i; + pending_read = m_config.m_mshr_type == SECTOR_TEX_FIFO? m_config.m_line_sz/SECTOR_SIZE : 0; } bool m_valid; unsigned m_rob_index; + unsigned pending_read; }; cache_stats m_stats; diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 63ba759..a8be4d2 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -150,6 +150,8 @@ void power_config::reg_options(class OptionParser * opp) void memory_config::reg_options(class OptionParser * opp) { + option_parser_register(opp, "-perf_sim_memcpy", OPT_BOOL, &m_perf_sim_memcpy, + "Fill the L2 cache on memcpy", "1"); option_parser_register(opp, "-gpgpu_dram_scheduler", OPT_INT32, &scheduler_type, "0 = fifo, 1 = FR-FCFS (defaul)", "1"); option_parser_register(opp, "-gpgpu_dram_partition_queues", OPT_CSTR, &gpgpu_L2_queue_config, @@ -202,7 +204,27 @@ void memory_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-dram_latency", OPT_UINT32, &dram_latency, "DRAM latency (default 30)", "30"); - + option_parser_register(opp, "-dual_bus_interface", OPT_UINT32, &dual_bus_interface, + "dual_bus_interface (default = 0) ", + "0"); + option_parser_register(opp, "-dram_bnk_indexing_policy", OPT_UINT32, &dram_bnk_indexing_policy, + "dram_bnk_indexing_policy (0 = normal indexing, 1 = Xoring with the higher bits) (Default = 0)", + "0"); + option_parser_register(opp, "-dram_bnkgrp_indexing_policy", OPT_UINT32, &dram_bnkgrp_indexing_policy, + "dram_bnkgrp_indexing_policy (0 = take higher bits, 1 = take lower bits) (Default = 0)", + "0"); + option_parser_register(opp, "-Seperate_Write_Queue_Enable", OPT_BOOL, &seperate_write_queue_enabled, + "Seperate_Write_Queue_Enable", + "0"); + option_parser_register(opp, "-Write_Queue_Size", OPT_CSTR, &write_queue_size_opt, + "Write_Queue_Size", + "32:28:16"); + option_parser_register(opp, "-Elimnate_rw_turnaround", OPT_BOOL, &elimnate_rw_turnaround, + "elimnate_rw_turnaround i.e set tWTR and tRTW = 0", + "0"); + option_parser_register(opp, "-icnt_flit_size", OPT_UINT32, &icnt_flit_size, + "icnt_flit_size", + "32"); m_address_mapping.addrdec_setoption(opp); } @@ -229,6 +251,12 @@ void shader_core_config::reg_options(class OptionParser * opp) "per-shader L1 data cache config " " {::,:::,::, | none}", "none" ); + option_parser_register(opp, "-l1_latency", OPT_UINT32, &m_L1D_config.l1_latency, + "L1 Hit Latency", + "0"); + option_parser_register(opp, "-smem_latency", OPT_UINT32, &smem_latency, + "smem Latency", + "3"); option_parser_register(opp, "-gpgpu_cache:dl1PrefL1", OPT_CSTR, &m_L1D_config.m_config_stringPrefL1, "per-shader L1 data cache config " " {::,:::,::, | none}", @@ -256,6 +284,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_shader_registers", OPT_UINT32, &gpgpu_shader_registers, "Number of registers per shader core. Limits number of concurrent CTAs. (default 8192)", "8192"); + option_parser_register(opp, "-gpgpu_ignore_resources_limitation", OPT_BOOL, &gpgpu_ignore_resources_limitation, + "gpgpu_ignore_resources_limitation (default 0)", + "0"); option_parser_register(opp, "-gpgpu_shader_cta", OPT_UINT32, &max_cta_per_core, "Maximum number of concurrent CTAs in shader (default 8)", "8"); @@ -277,6 +308,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_shmem_size", OPT_UINT32, &gpgpu_shmem_size, "Size of shared memory per shader core (default 16kB)", "16384"); + option_parser_register(opp, "-adpative_volta_cache_config", OPT_BOOL, &adpative_volta_cache_config, + "adpative_volta_cache_config", + "0"); option_parser_register(opp, "-gpgpu_shmem_size", OPT_UINT32, &gpgpu_shmem_sizeDefault, "Size of shared memory per shader core (default 16kB)", "16384"); @@ -295,6 +329,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_shmem_warp_parts", OPT_INT32, &mem_warp_parts, "Number of portions a warp is divided into for shared memory bank conflict check ", "2"); + option_parser_register(opp, "-gpgpu_shmem_warp_parts", OPT_INT32, &mem_warp_parts, + "Number of portions a warp is divided into for shared memory bank conflict check ", + "2"); option_parser_register(opp, "-gpgpu_warpdistro_shader", OPT_INT32, &gpgpu_warpdistro_shader, "Specify which shader core to collect the warp size distribution from", "-1"); @@ -303,7 +340,7 @@ void shader_core_config::reg_options(class OptionParser * opp) "0"); option_parser_register(opp, "-gpgpu_local_mem_map", OPT_BOOL, &gpgpu_local_mem_map, "Mapping from local memory space address to simulated GPU physical address space (default = enabled)", - "1"); + "1"); option_parser_register(opp, "-gpgpu_num_reg_banks", OPT_INT32, &gpgpu_num_reg_banks, "Number of register banks (default = 8)", "8"); @@ -313,6 +350,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_operand_collector_num_units_sp", OPT_INT32, &gpgpu_operand_collector_num_units_sp, "number of collector units (default = 4)", "4"); + option_parser_register(opp, "-gpgpu_operand_collector_num_units_dp", OPT_INT32, &gpgpu_operand_collector_num_units_dp, + "number of collector units (default = 0)", + "0"); option_parser_register(opp, "-gpgpu_operand_collector_num_units_sfu", OPT_INT32, &gpgpu_operand_collector_num_units_sfu, "number of collector units (default = 4)", "4"); @@ -328,6 +368,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_sp", OPT_INT32, &gpgpu_operand_collector_num_in_ports_sp, "number of collector unit in ports (default = 1)", "1"); + option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_dp", OPT_INT32, &gpgpu_operand_collector_num_in_ports_dp, + "number of collector unit in ports (default = 0)", + "0"); option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_sfu", OPT_INT32, &gpgpu_operand_collector_num_in_ports_sfu, "number of collector unit in ports (default = 1)", "1"); @@ -343,6 +386,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_sp", OPT_INT32, &gpgpu_operand_collector_num_out_ports_sp, "number of collector unit in ports (default = 1)", "1"); + option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_dp", OPT_INT32, &gpgpu_operand_collector_num_out_ports_dp, + "number of collector unit in ports (default = 0)", + "0"); option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_sfu", OPT_INT32, &gpgpu_operand_collector_num_out_ports_sfu, "number of collector unit in ports (default = 1)", "1"); @@ -356,27 +402,33 @@ void shader_core_config::reg_options(class OptionParser * opp) "number of collector unit in ports (default = 0)", "0"); option_parser_register(opp, "-gpgpu_coalesce_arch", OPT_INT32, &gpgpu_coalesce_arch, - "Coalescing arch (default = 13, anything else is off for now)", + "Coalescing arch (GT200 = 13, Fermi = 20)", "13"); option_parser_register(opp, "-gpgpu_num_sched_per_core", OPT_INT32, &gpgpu_num_sched_per_core, "Number of warp schedulers per core", "1"); option_parser_register(opp, "-gpgpu_max_insn_issue_per_warp", OPT_INT32, &gpgpu_max_insn_issue_per_warp, - "Max number of instructions that can be issued per warp in one cycle by scheduler", - "2"); + "Max number of instructions that can be issued per warp in one cycle by scheduler (either 1 or 2)", + "2"); + option_parser_register(opp, "-gpgpu_dual_issue_diff_exec_units", OPT_BOOL, &gpgpu_dual_issue_diff_exec_units, + "should dual issue use two different execution unit resources (Default = 1)", + "1"); option_parser_register(opp, "-gpgpu_simt_core_sim_order", OPT_INT32, &simt_core_sim_order, "Select the simulation order of cores in a cluster (0=Fix, 1=Round-Robin)", "1"); option_parser_register(opp, "-gpgpu_pipeline_widths", OPT_CSTR, &pipeline_widths_string, "Pipeline widths " - "ID_OC_SP,ID_OC_SFU,ID_OC_TENSOR_CORE,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_TENSOR_CORE,OC_EX_MEM,EX_WB", - "1,1,1,1,1,1,1,1,1,1" ); + "ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB,ID_OC_TENSOR_CORE,OC_EX_TENSOR_CORE", + "1,1,1,1,1,1,1,1,1,1,1" ); option_parser_register(opp, "-gpgpu_tensor_core_avail", OPT_INT32, &gpgpu_tensor_core_avail, "Tensor Core Available (default=0)", "0"); option_parser_register(opp, "-gpgpu_num_sp_units", OPT_INT32, &gpgpu_num_sp_units, "Number of SP units (default=1)", "1"); + option_parser_register(opp, "-gpgpu_num_dp_units", OPT_INT32, &gpgpu_num_dp_units, + "Number of DP units (default=0)", + "0"); option_parser_register(opp, "-gpgpu_num_sfu_units", OPT_INT32, &gpgpu_num_sfu_units, "Number of SF units (default=1)", "1"); @@ -425,7 +477,6 @@ void gpgpu_sim_config::reg_options(option_parser_t opp) option_parser_register(opp, "-gpgpu_flush_l2_cache", OPT_BOOL, &gpgpu_flush_l2_cache, "Flush L2 cache at the end of each kernel call", "0"); - option_parser_register(opp, "-gpgpu_deadlock_detect", OPT_BOOL, &gpu_deadlock_detect, "Stop the simulation at deadlock (1=on (default), 0=off)", "1"); @@ -669,7 +720,7 @@ gpgpu_sim::gpgpu_sim( const gpgpu_sim_config &config ) m_running_kernels.resize( config.max_concurrent_kernel, NULL ); m_last_issued_kernel = 0; - m_last_cluster_issue = 0; + m_last_cluster_issue = m_shader_config->n_simt_clusters-1; // this causes first launch to use simt cluster 0 *average_pipeline_duty_cycle=0; *active_sms=0; @@ -813,6 +864,7 @@ void gpgpu_sim::update_stats() { partiton_replys_in_parallel_total += partiton_replys_in_parallel; partiton_reqs_in_parallel_util_total += partiton_reqs_in_parallel_util; gpu_tot_sim_cycle_parition_util += gpu_sim_cycle_parition_util ; + gpu_tot_occupancy += gpu_occupancy; gpu_sim_cycle = 0; partiton_reqs_in_parallel = 0; @@ -821,6 +873,7 @@ void gpgpu_sim::update_stats() { gpu_sim_cycle_parition_util = 0; gpu_sim_insn = 0; m_total_cta_launched = 0; + gpu_occupancy = occupancy_stats(); } void gpgpu_sim::print_stats() @@ -994,6 +1047,9 @@ void gpgpu_sim::gpu_print_stat() printf("gpu_tot_sim_insn = %lld\n", gpu_tot_sim_insn+gpu_sim_insn); printf("gpu_tot_ipc = %12.4f\n", (float)(gpu_tot_sim_insn+gpu_sim_insn) / (gpu_tot_sim_cycle+gpu_sim_cycle)); printf("gpu_tot_issued_cta = %lld\n", gpu_tot_issued_cta + m_total_cta_launched); + printf("gpu_occupancy = %.4f\% \n", gpu_occupancy.get_occ_fraction() * 100); + printf("gpu_tot_occupancy = %.4f\% \n", (gpu_occupancy + gpu_tot_occupancy).get_occ_fraction() * 100); + extern unsigned long long g_max_total_param_size; fprintf(statfout, "max_total_param_size = %llu\n", g_max_total_param_size); @@ -1032,6 +1088,8 @@ void gpgpu_sim::gpu_print_stat() } printf("\nTotal_core_cache_stats:\n"); core_cache_stats.print_stats(stdout, "Total_core_cache_stats_breakdown"); + printf("\nTotal_core_cache_fail_stats:\n"); + core_cache_stats.print_fail_stats(stdout, "Total_core_cache_fail_stats_breakdown"); shader_print_scheduler_stat( stdout, false ); m_shader_stats->print(stdout); @@ -1076,6 +1134,8 @@ void gpgpu_sim::gpu_print_stat() printf("L2_total_cache_reservation_fails = %u\n", total_l2_css.res_fails); printf("L2_total_cache_breakdown:\n"); l2_stats.print_stats(stdout, "L2_cache_stats_breakdown"); + printf("L2_total_cache_reservation_fail_breakdown:\n"); + l2_stats.print_fail_stats(stdout, "L2_cache_stats_fail_breakdown"); total_l2_css.print_port_stats(stdout, "L2_cache"); } } @@ -1429,7 +1489,7 @@ void gpgpu_sim::cycle() if (mf) { unsigned response_size = mf->get_is_write()?mf->get_ctrl_size():mf->size(); if ( ::icnt_has_buffer( m_shader_config->mem2device(i), response_size ) ) { - if (!mf->get_is_write()) + //if (!mf->get_is_write()) mf->set_return_timestamp(gpu_sim_cycle+gpu_tot_sim_cycle); mf->set_status(IN_ICNT_TO_SHADER,gpu_sim_cycle+gpu_tot_sim_cycle); ::icnt_push( m_shader_config->mem2device(i), mf->get_tpc(), mf, response_size ); @@ -1462,12 +1522,14 @@ void gpgpu_sim::cycle() for (unsigned i=0;im_n_mem_sub_partition;i++) { //move memory request from interconnect into memory partition (if not backed up) //Note:This needs to be called in DRAM clock domain if there is no L2 cache in the system - if ( m_memory_sub_partition[i]->full() ) { + //In the worst case, we may need to push SECTOR_CHUNCK_SIZE requests, so ensure you have enough buffer for them + if ( m_memory_sub_partition[i]->full(SECTOR_CHUNCK_SIZE) ) { gpu_stall_dramfull++; } else { mem_fetch* mf = (mem_fetch*) icnt_pop( m_shader_config->mem2device(i) ); m_memory_sub_partition[i]->push( mf, gpu_sim_cycle + gpu_tot_sim_cycle ); - partiton_reqs_in_parallel_per_cycle++; + if(mf) + partiton_reqs_in_parallel_per_cycle++; } m_memory_sub_partition[i]->cache_cycle(gpu_sim_cycle+gpu_tot_sim_cycle); m_memory_sub_partition[i]->accumulate_L2cache_stats(m_power_stats->pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX]); @@ -1494,6 +1556,8 @@ void gpgpu_sim::cycle() // Update core icnt/cache stats for GPUWattch m_cluster[i]->get_icnt_stats(m_power_stats->pwr_mem_stat->n_simt_to_mem[CURRENT_STAT_IDX][i], m_power_stats->pwr_mem_stat->n_mem_to_simt[CURRENT_STAT_IDX][i]); m_cluster[i]->get_cache_stats(m_power_stats->pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX]); + m_cluster[i]->get_current_occupancy(gpu_occupancy.aggregate_warp_slot_filled, gpu_occupancy.aggregate_theoretical_warp_slots); + } float temp=0; for (unsigned i=0;inum_shader();i++){ @@ -1521,12 +1585,12 @@ void gpgpu_sim::cycle() issue_block2core(); - // Depending on configuration, flush the caches once all of threads are completed. + // Depending on configuration, invalidate the caches once all of threads are completed. int all_threads_complete = 1; if (m_config.gpgpu_flush_l1_cache) { for (unsigned i=0;in_simt_clusters;i++) { if (m_cluster[i]->get_not_completed() == 0) - m_cluster[i]->cache_flush(); + m_cluster[i]->cache_invalidate(); else all_threads_complete = 0 ; } @@ -1548,7 +1612,7 @@ void gpgpu_sim::cycle() int dlc = 0; for (unsigned i=0;im_n_mem;i++) { dlc = m_memory_sub_partition[i]->flushL2(); - assert (dlc == 0); // need to model actual writes to DRAM here + assert (dlc == 0); // TODO: need to model actual writes to DRAM here printf("Dirty lines flushed from L2 %d is %d\n", i, dlc ); } } @@ -1560,15 +1624,20 @@ void gpgpu_sim::cycle() time_t curr_time; time(&curr_time); unsigned long long elapsed_time = MAX(curr_time - g_simulation_starttime, 1); - if ( (elapsed_time - last_liveness_message_time) >= m_config.liveness_message_freq ) { + if ( (elapsed_time - last_liveness_message_time) >= m_config.liveness_message_freq && DTRACE(LIVENESS) ) { days = elapsed_time/(3600*24); hrs = elapsed_time/3600 - 24*days; minutes = elapsed_time/60 - 60*(hrs + 24*days); sec = elapsed_time - 60*(minutes + 60*(hrs + 24*days)); - - DPRINTF(LIVENESS, "GPGPU-Sim uArch: cycles simulated: %lld inst.: %lld (ipc=%4.1f) sim_rate=%u (inst/sec) elapsed = %u:%u:%02u:%02u / %s", - gpu_tot_sim_cycle + gpu_sim_cycle, gpu_tot_sim_insn + gpu_sim_insn, + + unsigned long long active = 0, total = 0; + for (unsigned i=0;in_simt_clusters;i++) { + m_cluster[i]->get_current_occupancy(active, total); + } + DPRINTF(LIVENESS, "uArch: inst.: %lld (ipc=%4.1f, occ=%0.4f\% [%llu / %llu]) sim_rate=%u (inst/sec) elapsed = %u:%u:%02u:%02u / %s", + gpu_tot_sim_insn + gpu_sim_insn, (double)gpu_sim_insn/(double)gpu_sim_cycle, + float(active)/float(total) * 100, active, total, (unsigned)((gpu_tot_sim_insn+gpu_sim_insn) / elapsed_time), (unsigned)days,(unsigned)hrs,(unsigned)minutes,(unsigned)sec, ctime(&curr_time)); @@ -1620,6 +1689,24 @@ void shader_core_ctx::dump_warp_state( FILE *fout ) const m_warp[w].print(fout); } + +void gpgpu_sim::perf_memcpy_to_gpu( size_t dst_start_addr, size_t count ) +{ + if (m_memory_config->m_perf_sim_memcpy) { + assert (dst_start_addr % 32 == 0); + + for ( unsigned counter = 0; counter < count; counter += 32 ) { + const unsigned wr_addr = dst_start_addr + counter; + addrdec_t raw_addr; + mem_access_sector_mask_t mask; + mask.set(wr_addr % 128 / 32); + m_memory_config->m_address_mapping.addrdec_tlx( wr_addr, &raw_addr ); + const unsigned partition_id = raw_addr.sub_partition / m_memory_config->m_n_sub_partition_per_memory_channel; + m_memory_partition_unit[ partition_id ]->handle_memcpy_to_gpu( wr_addr, raw_addr.sub_partition, mask ); + } + } +} + void gpgpu_sim::dump_pipeline( int mask, int s, int m ) const { /* diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h index 7d92c66..1bae1fa 100644 --- a/src/gpgpu-sim/gpu-sim.h +++ b/src/gpgpu-sim/gpu-sim.h @@ -198,8 +198,14 @@ struct memory_config { bk_tag_length = i-1; assert(nbkgrp>0 && "Number of bank groups cannot be zero"); tRCDWR = tRCD-(WL+1); + if(elimnate_rw_turnaround) + { + tRTW = 0; + tWTR = 0; + } else { tRTW = (CL+(BL/data_command_freq_ratio)+2-WL); - tWTR = (WL+(BL/data_command_freq_ratio)+tCDLR); + tWTR = (WL+(BL/data_command_freq_ratio)+tCDLR); + } tWTP = (WL+(BL/data_command_freq_ratio)+tWR); dram_atom_size = BL * busW * gpu_n_mem_per_ctrlr; // burst length x bus width x # chips per partition @@ -213,7 +219,9 @@ struct memory_config { m_L2_config.init(&m_address_mapping); m_valid = true; - icnt_flit_size = 32; // Default 32 + + sscanf(write_queue_size_opt,"%d:%d:%d", + &gpgpu_frfcfs_dram_write_queue_size,&write_high_watermark,&write_low_watermark); } void reg_options(class OptionParser * opp); @@ -264,12 +272,25 @@ struct memory_config { unsigned nbk; + bool elimnate_rw_turnaround; + unsigned data_command_freq_ratio; // frequency ratio between DRAM data bus and command bus (2 for GDDR3, 4 for GDDR5) unsigned dram_atom_size; // number of bytes transferred per read or write command linear_to_raw_address_translation m_address_mapping; unsigned icnt_flit_size; + + unsigned dram_bnk_indexing_policy; + unsigned dram_bnkgrp_indexing_policy; + bool dual_bus_interface; + + bool seperate_write_queue_enabled; + char *write_queue_size_opt; + unsigned gpgpu_frfcfs_dram_write_queue_size; + unsigned write_high_watermark; + unsigned write_low_watermark; + bool m_perf_sim_memcpy; }; // global counters and flags (please try not to add to this list!!!) @@ -362,6 +383,32 @@ private: friend class gpgpu_sim; }; +struct occupancy_stats { + occupancy_stats() : aggregate_warp_slot_filled(0), aggregate_theoretical_warp_slots(0){} + occupancy_stats( unsigned long long wsf, unsigned long long tws ) + : aggregate_warp_slot_filled(wsf), aggregate_theoretical_warp_slots(tws){} + + unsigned long long aggregate_warp_slot_filled; + unsigned long long aggregate_theoretical_warp_slots; + + float get_occ_fraction() const { + return float(aggregate_warp_slot_filled) / float(aggregate_theoretical_warp_slots); + } + + occupancy_stats& operator+=(const occupancy_stats& rhs) { + aggregate_warp_slot_filled += rhs.aggregate_warp_slot_filled; + aggregate_theoretical_warp_slots += rhs.aggregate_theoretical_warp_slots; + return *this; + } + + occupancy_stats operator+(const occupancy_stats& rhs) const{ + return occupancy_stats( aggregate_warp_slot_filled + rhs.aggregate_warp_slot_filled, + aggregate_theoretical_warp_slots + rhs.aggregate_theoretical_warp_slots + ); + } +}; + + class gpgpu_sim : public gpgpu_t { public: gpgpu_sim( const gpgpu_sim_config &config ); @@ -405,6 +452,8 @@ public: void gpu_print_stat(); void dump_pipeline( int mask, int s, int m ) const; + void perf_memcpy_to_gpu( size_t dst_start_addr, size_t count ); + //The next three functions added to be used by the functional simulation function //! Get shader core configuration @@ -498,6 +547,9 @@ public: unsigned long long gpu_tot_sim_insn; unsigned long long gpu_sim_insn_last_update; unsigned gpu_sim_insn_last_update_sid; + occupancy_stats gpu_occupancy; + occupancy_stats gpu_tot_occupancy; + FuncCache get_cache_config(std::string kernel_name); void set_cache_config(std::string kernel_name, FuncCache cacheConfig ); @@ -526,4 +578,5 @@ public: } }; + #endif diff --git a/src/gpgpu-sim/l2cache.cc b/src/gpgpu-sim/l2cache.cc index c5fc44e..25da107 100644 --- a/src/gpgpu-sim/l2cache.cc +++ b/src/gpgpu-sim/l2cache.cc @@ -74,6 +74,15 @@ memory_partition_unit::memory_partition_unit( unsigned partition_id, } } +void memory_partition_unit::handle_memcpy_to_gpu( size_t addr, unsigned global_subpart_id, mem_access_sector_mask_t mask ) +{ + unsigned p = global_sub_partition_id_to_local_id(global_subpart_id); + std::string mystring = + mask.to_string(); + MEMPART_DPRINTF("Copy Engine Request Received For Address=%llx, local_subpart=%u, global_subpart=%u, sector_mask=%s \n", addr, p, global_subpart_id, mystring.c_str()); + m_sub_partition[p]->force_l2_tag_update(addr,gpu_sim_cycle+gpu_tot_sim_cycle, mask); +} + memory_partition_unit::~memory_partition_unit() { delete m_dram; @@ -93,7 +102,9 @@ memory_partition_unit::arbitration_metadata::arbitration_metadata(const struct m m_private_credit_limit = 1; m_shared_credit_limit = config->gpgpu_frfcfs_dram_sched_queue_size + config->gpgpu_dram_return_queue_size - - (config->m_n_sub_partition_per_memory_channel - 1); + - (config->m_n_sub_partition_per_memory_channel - 1); + if(config->seperate_write_queue_enabled ) + m_shared_credit_limit += config->gpgpu_frfcfs_dram_write_queue_size; if (config->gpgpu_frfcfs_dram_sched_queue_size == 0 or config->gpgpu_dram_return_queue_size == 0) { @@ -220,7 +231,8 @@ void memory_partition_unit::dram_cycle() m_dram->cycle(); m_dram->dram_log(SAMPLELOG); - if( !m_dram->full() ) { + // mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top(); + //if( !m_dram->full(mf->is_write()) ) { // L2->DRAM queue to DRAM latency queue // Arbitrate among multiple L2 subpartitions int last_issued_partition = m_arbitration_metadata.last_borrower(); @@ -228,6 +240,9 @@ void memory_partition_unit::dram_cycle() int spid = (p + last_issued_partition + 1) % m_config->m_n_sub_partition_per_memory_channel; if (!m_sub_partition[spid]->L2_dram_queue_empty() && can_issue_to_dram(spid)) { mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top(); + if(m_dram->full(mf->is_write()) ) + break; + m_sub_partition[spid]->L2_dram_queue_pop(); MEMPART_DPRINTF("Issue mem_fetch request %p from sub partition %d to dram\n", mf, spid); dram_delay_t d; @@ -239,12 +254,13 @@ void memory_partition_unit::dram_cycle() break; // the DRAM should only accept one request per cycle } } - } + //} // DRAM latency queue - if( !m_dram_latency_queue.empty() && ( (gpu_sim_cycle+gpu_tot_sim_cycle) >= m_dram_latency_queue.front().ready_cycle ) && !m_dram->full() ) { - mem_fetch* mf = m_dram_latency_queue.front().req; - m_dram_latency_queue.pop_front(); + + if( !m_dram_latency_queue.empty() && ( (gpu_sim_cycle+gpu_tot_sim_cycle) >= m_dram_latency_queue.front().ready_cycle ) && !m_dram->full(m_dram_latency_queue.front().req->is_write()) ) { + mem_fetch* mf = m_dram_latency_queue.front().req; + m_dram_latency_queue.pop_front(); m_dram->push(mf); } } @@ -299,6 +315,7 @@ memory_sub_partition::memory_sub_partition( unsigned sub_partition_id, m_id = sub_partition_id; m_config=config; m_stats=stats; + m_memcpy_cycle_offset = 0; assert(m_id < m_config->m_n_mem_sub_partition); @@ -343,6 +360,14 @@ void memory_sub_partition::cache_cycle( unsigned cycle ) mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); m_L2_icnt_queue->push(mf); }else{ + if(m_config->m_L2_config.m_write_alloc_policy == FETCH_ON_WRITE) + { + mem_fetch* original_wr_mf = mf->get_original_wr_mf(); + assert(original_wr_mf); + original_wr_mf->set_reply(); + original_wr_mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); + m_L2_icnt_queue->push(original_wr_mf); + } m_request_tracker.erase(mf); delete mf; } @@ -355,10 +380,11 @@ void memory_sub_partition::cache_cycle( unsigned cycle ) if ( !m_config->m_L2_config.disabled() && m_L2cache->waiting_for_fill(mf) ) { if (m_L2cache->fill_port_free()) { mf->set_status(IN_PARTITION_L2_FILL_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); - m_L2cache->fill(mf,gpu_sim_cycle+gpu_tot_sim_cycle); + m_L2cache->fill(mf,gpu_sim_cycle+gpu_tot_sim_cycle+m_memcpy_cycle_offset); m_dram_L2_queue->pop(); } } else if ( !m_L2_icnt_queue->full() ) { + if(mf->is_write() && mf->get_type() == WRITE_ACK) mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); m_L2_icnt_queue->push(mf); m_dram_L2_queue->pop(); @@ -380,9 +406,10 @@ void memory_sub_partition::cache_cycle( unsigned cycle ) bool port_free = m_L2cache->data_port_free(); if ( !output_full && port_free ) { std::list events; - enum cache_request_status status = m_L2cache->access(mf->get_addr(),mf,gpu_sim_cycle+gpu_tot_sim_cycle,events); + enum cache_request_status status = m_L2cache->access(mf->get_addr(),mf,gpu_sim_cycle+gpu_tot_sim_cycle+m_memcpy_cycle_offset,events); bool write_sent = was_write_sent(events); bool read_sent = was_read_sent(events); + MEM_SUBPART_DPRINTF("Probing L2 cache Address=%llx, status=%u\n", mf->get_addr(), status); if ( status == HIT ) { if( !write_sent ) { @@ -402,6 +429,11 @@ void memory_sub_partition::cache_cycle( unsigned cycle ) m_icnt_L2_queue->pop(); } } else if ( status != RESERVATION_FAIL ) { + if(mf->is_write() && (m_config->m_L2_config.m_write_alloc_policy == FETCH_ON_WRITE || m_config->m_L2_config.m_write_alloc_policy == LAZY_FETCH_ON_READ) && !was_writeallocate_sent(events)) { + mf->set_reply(); + mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); + m_L2_icnt_queue->push(mf); + } // L2 cache accepted request m_icnt_L2_queue->pop(); } else { @@ -432,6 +464,11 @@ bool memory_sub_partition::full() const return m_icnt_L2_queue->full(); } +bool memory_sub_partition::full(unsigned size) const +{ + return m_icnt_L2_queue->is_avilable_size(size); +} + bool memory_sub_partition::L2_dram_queue_empty() const { return m_L2_dram_queue->empty(); @@ -532,7 +569,15 @@ unsigned memory_sub_partition::flushL2() if (!m_config->m_L2_config.disabled()) { m_L2cache->flush(); } - return 0; // L2 is read only in this version + return 0; //TODO: write the flushed data to the main memory +} + +unsigned memory_sub_partition::invalidateL2() +{ + if (!m_config->m_L2_config.disabled()) { + m_L2cache->invalidate(); + } + return 0; } bool memory_sub_partition::busy() const @@ -540,21 +585,94 @@ bool memory_sub_partition::busy() const return !m_request_tracker.empty(); } -void memory_sub_partition::push( mem_fetch* req, unsigned long long cycle ) -{ - if (req) { - m_request_tracker.insert(req); - m_stats->memlatstat_icnt2mem_pop(req); - if( req->istexture() ) { - m_icnt_L2_queue->push(req); - req->set_status(IN_PARTITION_ICNT_TO_L2_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); - } else { - rop_delay_t r; - r.req = req; - r.ready_cycle = cycle + m_config->rop_latency; - m_rop.push(r); - req->set_status(IN_PARTITION_ROP_DELAY,gpu_sim_cycle+gpu_tot_sim_cycle); - } +std::vector memory_sub_partition::breakdown_request_to_sector_requests(mem_fetch* mf) +{ + std::vector result; + + if(mf->get_data_size() == SECTOR_SIZE && mf->get_access_sector_mask().count() == 1) { + result.push_back(mf); + } else if (mf->get_data_size() == 128 || mf->get_data_size() == 64) { + //We only accept 32, 64 and 128 bytes reqs + unsigned start=0, end=0; + if(mf->get_data_size() == 128) { + start=0; end=3; + } else if (mf->get_data_size() == 64 && mf->get_access_sector_mask().to_string() == "1100") { + start=2; end=3; + } else if (mf->get_data_size() == 64 && mf->get_access_sector_mask().to_string() == "0011") { + start=0; end=1; + } else if (mf->get_data_size() == 64 && (mf->get_access_sector_mask().to_string() == "1111" || mf->get_access_sector_mask().to_string() == "0000")) { + if(mf->get_addr() % 128 == 0) { + start=0; end=1; + } else { + start=2; end=3; + } + } else + { + printf("Invalid sector received, address = 0x%06x, sector mask = %s, data size = %d", + mf->get_addr(), mf->get_access_sector_mask(), mf->get_data_size()); + assert(0 && "Undefined sector mask is received"); + } + + std::bitset byte_sector_mask; + byte_sector_mask.reset(); + for(unsigned k=start*SECTOR_SIZE; k< SECTOR_SIZE; ++k) + byte_sector_mask.set(k); + + for(unsigned j=start, i=0; j<= end ; ++j, ++i){ + + const mem_access_t *ma = new mem_access_t( mf->get_access_type(), + mf->get_addr() + SECTOR_SIZE*i, + SECTOR_SIZE, + mf->is_write(), + mf->get_access_warp_mask(), + mf->get_access_byte_mask() & byte_sector_mask, + std::bitset().set(j)); + + mem_fetch *n_mf = new mem_fetch( *ma, + NULL, + mf->get_ctrl_size(), + mf->get_wid(), + mf->get_sid(), + mf->get_tpc(), + mf->get_mem_config(), + mf); + + result.push_back(n_mf); + byte_sector_mask <<= SECTOR_SIZE; + } + } else { + printf("Invalid sector received, address = 0x%06x, sector mask = %d, byte mask = , data size = %d", + mf->get_addr(), mf->get_access_sector_mask().count(), mf->get_data_size()); + assert(0 && "Undefined data size is received"); + } + + return result; +} + +void memory_sub_partition::push( mem_fetch* m_req, unsigned long long cycle ) +{ + if (m_req) { + m_stats->memlatstat_icnt2mem_pop(m_req); + std::vector reqs; + if(m_config->m_L2_config.m_cache_type == SECTOR) + reqs = breakdown_request_to_sector_requests(m_req); + else + reqs.push_back(m_req); + + for(unsigned i=0; iistexture() ) { + m_icnt_L2_queue->push(req); + req->set_status(IN_PARTITION_ICNT_TO_L2_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); + } else { + rop_delay_t r; + r.req = req; + r.ready_cycle = cycle + m_config->rop_latency; + m_rop.push(r); + req->set_status(IN_PARTITION_ROP_DELAY,gpu_sim_cycle+gpu_tot_sim_cycle); + } + } } } diff --git a/src/gpgpu-sim/l2cache.h b/src/gpgpu-sim/l2cache.h index 3df54b1..18c0a8b 100644 --- a/src/gpgpu-sim/l2cache.h +++ b/src/gpgpu-sim/l2cache.h @@ -72,6 +72,7 @@ public: void print_stat( FILE *fp ) { m_dram->print_stat(fp); } void visualize() const { m_dram->visualize(); } void print( FILE *fp ) const; + void handle_memcpy_to_gpu( size_t dst_start_addr, unsigned subpart_id, mem_access_sector_mask_t mask ); class memory_sub_partition * get_sub_partition(int sub_partition_id) { @@ -154,12 +155,14 @@ public: void cache_cycle( unsigned cycle ); bool full() const; + bool full(unsigned size) const; void push( class mem_fetch* mf, unsigned long long clock_cycle ); class mem_fetch* pop(); class mem_fetch* top(); void set_done( mem_fetch *mf ); unsigned flushL2(); + unsigned invalidateL2(); // interface to L2_dram_queue bool L2_dram_queue_empty() const; @@ -177,6 +180,12 @@ public: void accumulate_L2cache_stats(class cache_stats &l2_stats) const; void get_L2cache_sub_stats(struct cache_sub_stats &css) const; + void force_l2_tag_update(new_addr_type addr, unsigned time, mem_access_sector_mask_t mask) + { + m_L2cache->force_tag_access( addr, m_memcpy_cycle_offset + time, mask ); + m_memcpy_cycle_offset += 1; + } + private: // data unsigned m_id; //< the global sub partition ID @@ -207,6 +216,15 @@ private: std::set m_request_tracker; friend class L2interface; + + std::vector breakdown_request_to_sector_requests(mem_fetch* mf); + + // This is a cycle offset that has to be applied to the l2 accesses to account for + // the cudamemcpy read/writes. We want GPGPU-Sim to only count cycles for kernel execution + // but we want cudamemcpy to go through the L2. Everytime an access is made from cudamemcpy + // this counter is incremented, and when the l2 is accessed (in both cudamemcpyies and otherwise) + // this value is added to the gpgpu-sim cycle counters. + unsigned m_memcpy_cycle_offset; }; class L2interface : public mem_fetch_interface { diff --git a/src/gpgpu-sim/l2cache_trace.h b/src/gpgpu-sim/l2cache_trace.h index 3dac87d..2235cdc 100644 --- a/src/gpgpu-sim/l2cache_trace.h +++ b/src/gpgpu-sim/l2cache_trace.h @@ -34,6 +34,9 @@ #define MEMPART_PRINT_STR SIM_PRINT_STR " %d - " #define MEMPART_DTRACE(x) ( DTRACE(x) && (Trace::sampling_memory_partition == -1 || Trace::sampling_memory_partition == (int)get_mpid()) ) +#define MEM_SUBPART_PRINT_STR SIM_PRINT_STR " %d - " +#define MEM_SUBPART_DTRACE(x) ( DTRACE(x) && (Trace::sampling_memory_partition == -1 || Trace::sampling_memory_partition == (int)m_id) ) + // Intended to be called from inside components of a memory partition // Depends on a get_mpid() function #define MEMPART_DPRINTF(...) do {\ @@ -46,10 +49,23 @@ }\ } while (0) +#define MEM_SUBPART_DPRINTF(...) do {\ + if (MEM_SUBPART_DTRACE(MEMORY_PARTITION_UNIT)) {\ + printf( MEM_SUBPART_PRINT_STR,\ + gpu_sim_cycle + gpu_tot_sim_cycle,\ + Trace::trace_streams_str[Trace::MEMORY_SUBPARTITION_UNIT],\ + m_id );\ + printf(__VA_ARGS__);\ + }\ +} while (0) + #else #define MEMPART_DTRACE(x) (false) #define MEMPART_DPRINTF(x, ...) do {} while (0) +#define MEM_SUBPART_DTRACE(x) (false) +#define MEM_SUBPART_DPRINTF(x, ...) do {} while (0) + #endif diff --git a/src/gpgpu-sim/mem_fetch.cc b/src/gpgpu-sim/mem_fetch.cc index 729636d..a260a35 100644 --- a/src/gpgpu-sim/mem_fetch.cc +++ b/src/gpgpu-sim/mem_fetch.cc @@ -39,7 +39,10 @@ mem_fetch::mem_fetch( const mem_access_t &access, unsigned wid, unsigned sid, unsigned tpc, - const struct memory_config *config ) + const struct memory_config *config, + mem_fetch *m_original_mf, + mem_fetch *m_original_wr_mf) + { m_request_uid = sm_next_mf_request_uid++; m_access = access; @@ -61,6 +64,8 @@ mem_fetch::mem_fetch( const mem_access_t &access, m_status_change = gpu_sim_cycle + gpu_tot_sim_cycle; m_mem_config = config; icnt_flit_size = config->icnt_flit_size; + original_mf = m_original_mf; + original_wr_mf = m_original_wr_mf; } mem_fetch::~mem_fetch() diff --git a/src/gpgpu-sim/mem_fetch.h b/src/gpgpu-sim/mem_fetch.h index de98748..e5efffd 100644 --- a/src/gpgpu-sim/mem_fetch.h +++ b/src/gpgpu-sim/mem_fetch.h @@ -55,7 +55,9 @@ public: unsigned wid, unsigned sid, unsigned tpc, - const struct memory_config *config ); + const struct memory_config *config, + mem_fetch *original_mf = NULL, + mem_fetch *original_wr_mf = NULL); ~mem_fetch(); void set_status( enum mem_fetch_status status, unsigned long long cycle ); @@ -104,6 +106,7 @@ public: enum mem_access_type get_access_type() const { return m_access.get_type(); } const active_mask_t& get_access_warp_mask() const { return m_access.get_warp_mask(); } mem_access_byte_mask_t get_access_byte_mask() const { return m_access.get_byte_mask(); } + mem_access_sector_mask_t get_access_sector_mask() const { return m_access.get_sector_mask(); } address_type get_pc() const { return m_inst.empty()?-1:m_inst.pc; } const warp_inst_t &get_inst() { return m_inst; } @@ -112,6 +115,10 @@ public: const memory_config *get_mem_config(){return m_mem_config;} unsigned get_num_flits(bool simt_to_mem); + + mem_fetch* get_original_mf() { return original_mf; } + mem_fetch* get_original_wr_mf() { return original_wr_mf; } + private: // request source information unsigned m_request_uid; @@ -143,6 +150,10 @@ private: const struct memory_config *m_mem_config; unsigned icnt_flit_size; + + mem_fetch* original_mf; //this pointer is set up when a request is divided into sector requests at L2 cache (if the req size > L2 sector size), so the pointer refers to the original request + mem_fetch* original_wr_mf; //this pointer refers to the original write req, when fetch-on-write policy is used + }; #endif diff --git a/src/gpgpu-sim/mem_latency_stat.cc b/src/gpgpu-sim/mem_latency_stat.cc index fde0eff..c5452b9 100644 --- a/src/gpgpu-sim/mem_latency_stat.cc +++ b/src/gpgpu-sim/mem_latency_stat.cc @@ -75,6 +75,10 @@ memory_stats_t::memory_stats_t( unsigned n_shader, const struct shader_core_conf max_mf_latency = 0; max_icnt2mem_latency = 0; max_icnt2sh_latency = 0; + tot_icnt2mem_latency = 0; + tot_icnt2sh_latency = 0; + tot_mrq_num = 0; + tot_mrq_latency = 0; memset(mrq_lat_table, 0, sizeof(unsigned)*32); memset(dq_lat_table, 0, sizeof(unsigned)*32); memset(mf_lat_table, 0, sizeof(unsigned)*32); @@ -158,6 +162,7 @@ void memory_stats_t::memlatstat_read_done(mem_fetch *mf) mf_max_lat_table[mf->get_tlx_addr().chip][mf->get_tlx_addr().bk] = mf_latency; unsigned icnt2sh_latency; icnt2sh_latency = (gpu_tot_sim_cycle+gpu_sim_cycle) - mf->get_return_timestamp(); + tot_icnt2sh_latency += icnt2sh_latency; icnt2sh_lat_table[LOGB2(icnt2sh_latency)]++; if (icnt2sh_latency > max_icnt2sh_latency) max_icnt2sh_latency = icnt2sh_latency; @@ -191,6 +196,7 @@ void memory_stats_t::memlatstat_icnt2mem_pop(mem_fetch *mf) if (m_memory_config->gpgpu_memlatency_stat) { unsigned icnt2mem_latency; icnt2mem_latency = (gpu_tot_sim_cycle+gpu_sim_cycle) - mf->get_timestamp(); + tot_icnt2mem_latency += icnt2mem_latency; icnt2mem_lat_table[LOGB2(icnt2mem_latency)]++; if (icnt2mem_latency > max_icnt2mem_latency) max_icnt2mem_latency = icnt2mem_latency; @@ -216,14 +222,19 @@ void memory_stats_t::memlatstat_print( unsigned n_mem, unsigned gpu_mem_n_bk ) unsigned max_bank_accesses, min_bank_accesses, max_chip_accesses, min_chip_accesses; if (m_memory_config->gpgpu_memlatency_stat) { + printf("maxmflatency = %d \n", max_mf_latency); + printf("max_icnt2mem_latency = %d \n", max_icnt2mem_latency); printf("maxmrqlatency = %d \n", max_mrq_latency); - printf("maxdqlatency = %d \n", max_dq_latency); - printf("maxmflatency = %d \n", max_mf_latency); + //printf("maxdqlatency = %d \n", max_dq_latency); + printf("max_icnt2sh_latency = %d \n", max_icnt2sh_latency); if (num_mfs) { printf("averagemflatency = %lld \n", mf_total_lat/num_mfs); + printf("avg_icnt2mem_latency = %lld \n", tot_icnt2mem_latency/num_mfs); + if(tot_mrq_num) + printf("avg_mrq_latency = %lld \n", tot_mrq_latency/tot_mrq_num); + + printf("avg_icnt2sh_latency = %lld \n", tot_icnt2sh_latency/num_mfs); } - printf("max_icnt2mem_latency = %d \n", max_icnt2mem_latency); - printf("max_icnt2sh_latency = %d \n", max_icnt2sh_latency); printf("mrq_lat_table:"); for (i=0; i< 32; i++) { printf("%d \t", mrq_lat_table[i]); diff --git a/src/gpgpu-sim/mem_latency_stat.h b/src/gpgpu-sim/mem_latency_stat.h index 4968a3b..5b89202 100644 --- a/src/gpgpu-sim/mem_latency_stat.h +++ b/src/gpgpu-sim/mem_latency_stat.h @@ -56,6 +56,10 @@ public: unsigned max_dq_latency; unsigned max_mf_latency; unsigned max_icnt2mem_latency; + unsigned long long int tot_icnt2mem_latency; + unsigned long long int tot_icnt2sh_latency; + unsigned long long int tot_mrq_latency; + unsigned long long int tot_mrq_num; unsigned max_icnt2sh_latency; unsigned mrq_lat_table[32]; unsigned dq_lat_table[32]; diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index f9cfa58..82f9181 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -74,13 +74,14 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, shader_core_stats *stats ) : core_t( gpu, NULL, config->warp_size, config->n_thread_per_shader ), m_barriers( this, config->max_warps_per_shader, config->max_cta_per_core, config->max_barriers_per_cta, config->warp_size ), - m_dynamic_warp_id(0) + m_dynamic_warp_id(0), m_active_warps(0) { m_cluster = cluster; m_config = config; m_memory_config = mem_config; m_stats = stats; unsigned warp_size=config->warp_size; + Issue_Prio = 0; m_sid = shader_id; m_tpc = tpc_id; @@ -131,6 +132,8 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, CONCRETE_SCHEDULER_TWO_LEVEL_ACTIVE : sched_config.find("gto") != std::string::npos ? CONCRETE_SCHEDULER_GTO : + sched_config.find("old") != std::string::npos ? + CONCRETE_SCHEDULER_OLDEST_FIRST : sched_config.find("warp_limiting") != std::string::npos ? CONCRETE_SCHEDULER_WARP_LIMITING: NUM_CONCRETE_SCHEDULERS; @@ -147,6 +150,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_simt_stack, &m_warp, &m_pipeline_reg[ID_OC_SP], + &m_pipeline_reg[ID_OC_DP], &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], @@ -162,6 +166,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_simt_stack, &m_warp, &m_pipeline_reg[ID_OC_SP], + &m_pipeline_reg[ID_OC_DP], &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], @@ -178,6 +183,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_simt_stack, &m_warp, &m_pipeline_reg[ID_OC_SP], + &m_pipeline_reg[ID_OC_DP], &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], @@ -185,6 +191,22 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, ) ); break; + case CONCRETE_SCHEDULER_OLDEST_FIRST: + schedulers.push_back( + new oldest_scheduler( m_stats, + this, + m_scoreboard, + m_simt_stack, + &m_warp, + &m_pipeline_reg[ID_OC_SP], + &m_pipeline_reg[ID_OC_DP], + &m_pipeline_reg[ID_OC_SFU], + &m_pipeline_reg[ID_OC_TENSOR_CORE], + &m_pipeline_reg[ID_OC_MEM], + i + ) + ); + break; case CONCRETE_SCHEDULER_WARP_LIMITING: schedulers.push_back( new swl_scheduler( m_stats, @@ -193,6 +215,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_simt_stack, &m_warp, &m_pipeline_reg[ID_OC_SP], + &m_pipeline_reg[ID_OC_DP], &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], @@ -215,8 +238,9 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, } //op collector configuration - enum { SP_CUS, SFU_CUS, TENSOR_CORE_CUS, MEM_CUS, GEN_CUS }; + enum { SP_CUS, DP_CUS, SFU_CUS, MEM_CUS, TENSOR_CORE_CUS, GEN_CUS }; m_operand_collector.add_cu_set(SP_CUS, m_config->gpgpu_operand_collector_num_units_sp, m_config->gpgpu_operand_collector_num_out_ports_sp); + m_operand_collector.add_cu_set(DP_CUS, m_config->gpgpu_operand_collector_num_units_dp, m_config->gpgpu_operand_collector_num_out_ports_dp); m_operand_collector.add_cu_set(SFU_CUS, m_config->gpgpu_operand_collector_num_units_sfu, m_config->gpgpu_operand_collector_num_out_ports_sfu); m_operand_collector.add_cu_set(TENSOR_CORE_CUS, config->gpgpu_operand_collector_num_units_tensor_core, config->gpgpu_operand_collector_num_out_ports_tensor_core); m_operand_collector.add_cu_set(MEM_CUS, m_config->gpgpu_operand_collector_num_units_mem, m_config->gpgpu_operand_collector_num_out_ports_mem); @@ -234,6 +258,15 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, in_ports.clear(),out_ports.clear(),cu_sets.clear(); } + for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_dp; i++) { + in_ports.push_back(&m_pipeline_reg[ID_OC_DP]); + out_ports.push_back(&m_pipeline_reg[OC_EX_DP]); + cu_sets.push_back((unsigned)DP_CUS); + cu_sets.push_back((unsigned)GEN_CUS); + m_operand_collector.add_port(in_ports,out_ports,cu_sets); + in_ports.clear(),out_ports.clear(),cu_sets.clear(); + } + for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_sfu; i++) { in_ports.push_back(&m_pipeline_reg[ID_OC_SFU]); out_ports.push_back(&m_pipeline_reg[OC_EX_SFU]); @@ -279,7 +312,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_operand_collector.init( m_config->gpgpu_num_reg_banks, this ); // execute - m_num_function_units = m_config->gpgpu_num_sp_units + m_config->gpgpu_num_sfu_units + config->gpgpu_num_tensor_core_units + 1; // sp_unit, sfu, ldst_unit + m_num_function_units = m_config->gpgpu_num_sp_units + m_config->gpgpu_num_dp_units + m_config->gpgpu_num_sfu_units + m_config->gpgpu_num_tensor_core_units + 1; // sp_unit, sfu, ldst_unit //m_dispatch_port = new enum pipeline_stage_name_t[ m_num_function_units ]; //m_issue_port = new enum pipeline_stage_name_t[ m_num_function_units ]; @@ -291,6 +324,12 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_issue_port.push_back(OC_EX_SP); } + for (int k = 0; k < m_config->gpgpu_num_dp_units; k++) { + m_fu.push_back(new dp_unit( &m_pipeline_reg[EX_WB], m_config, this )); + m_dispatch_port.push_back(ID_OC_DP); + m_issue_port.push_back(OC_EX_DP); + } + for (int k = 0; k < m_config->gpgpu_num_sfu_units; k++) { m_fu.push_back(new sfu( &m_pipeline_reg[EX_WB], m_config, this )); m_dispatch_port.push_back(ID_OC_SFU); @@ -307,7 +346,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_fu.push_back(m_ldst_unit); m_dispatch_port.push_back(ID_OC_MEM); m_issue_port.push_back(OC_EX_MEM); - + assert(m_num_function_units == m_fu.size() and m_fu.size() == m_dispatch_port.size() and m_fu.size() == m_issue_port.size()); //there are as many result buses as the width of the EX_WB stage @@ -341,6 +380,7 @@ void shader_core_ctx::reinit(unsigned start_thread, unsigned end_thread, bool re m_occupied_ctas = 0; m_occupied_hwtid.reset(); m_occupied_cta_to_hwtid.clear(); + m_active_warps = 0; } for (unsigned i = start_thread; iget_pdom_stack_top_info(pc,rpc); } +float shader_core_ctx::get_current_occupancy( unsigned long long & active, unsigned long long & total ) const +{ + // To match the achieved_occupancy in nvprof, only SMs that are active are counted toward the occupancy. + if ( m_active_warps > 0 ) { + total += m_warp.size(); + active += m_active_warps; + return float(active) / float(total); + } else { + return 0; + } +} + void shader_core_stats::print( FILE* fout ) const { unsigned long long thread_icount_uarch=0; @@ -436,15 +489,15 @@ void shader_core_stats::print( FILE* fout ) const fprintf(fout, "gpgpu_n_intrawarp_mshr_merge = %d\n", gpgpu_n_intrawarp_mshr_merge); fprintf(fout, "gpgpu_n_cmem_portconflict = %d\n", gpgpu_n_cmem_portconflict); - fprintf(fout, "gpgpu_stall_shd_mem[c_mem][bk_conf] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][BK_CONF]); - fprintf(fout, "gpgpu_stall_shd_mem[c_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][MSHR_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[c_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[c_mem][data_port_stall] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][DATA_PORT_STALL]); - fprintf(fout, "gpgpu_stall_shd_mem[t_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][MSHR_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[t_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[t_mem][data_port_stall] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][DATA_PORT_STALL]); + fprintf(fout, "gpgpu_stall_shd_mem[c_mem][resource_stall] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][BK_CONF]); + //fprintf(fout, "gpgpu_stall_shd_mem[c_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][MSHR_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[c_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[c_mem][data_port_stall] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][DATA_PORT_STALL]); + //fprintf(fout, "gpgpu_stall_shd_mem[t_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][MSHR_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[t_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[t_mem][data_port_stall] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][DATA_PORT_STALL]); fprintf(fout, "gpgpu_stall_shd_mem[s_mem][bk_conf] = %d\n", gpu_stall_shd_mem_breakdown[S_MEM][BK_CONF]); - fprintf(fout, "gpgpu_stall_shd_mem[gl_mem][bk_conf] = %d\n", + fprintf(fout, "gpgpu_stall_shd_mem[gl_mem][resource_stall] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][BK_CONF] + gpu_stall_shd_mem_breakdown[G_MEM_ST][BK_CONF] + gpu_stall_shd_mem_breakdown[L_MEM_LD][BK_CONF] + @@ -462,22 +515,22 @@ void shader_core_stats::print( FILE* fout ) const gpu_stall_shd_mem_breakdown[L_MEM_LD][DATA_PORT_STALL] + gpu_stall_shd_mem_breakdown[L_MEM_ST][DATA_PORT_STALL] ); // data port stall at data cache - fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][MSHR_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_CACHE_RSRV_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][MSHR_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_CACHE_RSRV_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][MSHR_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_CACHE_RSRV_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][MSHR_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_CACHE_RSRV_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][MSHR_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_CACHE_RSRV_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][MSHR_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_CACHE_RSRV_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][MSHR_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_CACHE_RSRV_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][MSHR_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_CACHE_RSRV_FAIL]); fprintf(fout, "gpu_reg_bank_conflict_stalls = %d\n", gpu_reg_bank_conflict_stalls); @@ -488,6 +541,14 @@ void shader_core_stats::print( FILE* fout ) const for (unsigned i = 3; i < m_config->warp_size + 3; i++) fprintf(fout, "\tW%d:%d", i-2, shader_cycle_distro[i]); fprintf(fout, "\n"); + fprintf(fout, "single_issue_nums: "); + for (unsigned i = 0; i < m_config->gpgpu_num_sched_per_core; i++) + fprintf(fout, "WS%d:%d\t", i, single_issue_nums[i]); + fprintf(fout, "\n"); + fprintf(fout, "dual_issue_nums: "); + for (unsigned i = 0; i < m_config->gpgpu_num_sched_per_core; i++) + fprintf(fout, "WS%d:%d\t", i, dual_issue_nums[i]); + fprintf(fout, "\n"); m_outgoing_traffic_stats->print(fout); m_incoming_traffic_stats->print(fout); @@ -669,13 +730,15 @@ void shader_core_ctx::fetch() } if( did_exit ) m_warp[warp_id].set_done_exit(); + --m_active_warps; + assert(m_active_warps >= 0); } // this code fetches instructions from the i-cache or generates memory requests if( !m_warp[warp_id].functional_done() && !m_warp[warp_id].imiss_pending() && m_warp[warp_id].ibuffer_empty() ) { address_type pc = m_warp[warp_id].get_pc(); address_type ppc = pc + PROGRAM_MEM_START; - unsigned nbytes=16; + unsigned nbytes=16; unsigned offset_in_block = pc & (m_config->m_L1I_config.get_line_sz()-1); if( (offset_in_block+nbytes) > m_config->m_L1I_config.get_line_sz() ) nbytes = (m_config->m_L1I_config.get_line_sz()-offset_in_block); @@ -750,10 +813,19 @@ void shader_core_ctx::issue_warp( register_set& pipe_reg_set, const warp_inst_t* } void shader_core_ctx::issue(){ + + //Ensure fair round robin issu between schedulers + unsigned j; + for (unsigned i = 0; i < schedulers.size(); i++) { + j = (Issue_Prio + i) % schedulers.size(); + schedulers[j]->cycle(); + } + Issue_Prio = (Issue_Prio+1)% schedulers.size(); + //really is issue; - for (unsigned i = 0; i < schedulers.size(); i++) { - schedulers[i]->cycle(); - } + //for (unsigned i = 0; i < schedulers.size(); i++) { + // schedulers[i]->cycle(); + //} } shd_warp_t& scheduler_unit::warp(int i){ @@ -868,7 +940,10 @@ void scheduler_unit::cycle() unsigned warp_id = (*iter)->get_warp_id(); unsigned checked=0; unsigned issued=0; - unsigned max_issue = m_shader->m_config->gpgpu_max_insn_issue_per_warp; + exec_unit_type_t previous_issued_inst_exec_type = exec_unit_type_t::NONE; + unsigned max_issue = m_shader->m_config->gpgpu_max_insn_issue_per_warp; + bool diff_exec_units = m_shader->m_config->gpgpu_dual_issue_diff_exec_units; + while( !warp(warp_id).waiting() && !warp(warp_id).ibuffer_empty() && (checked < max_issue) && (checked <= issued) && (issued < max_issue) ) { const warp_inst_t *pI = warp(warp_id).ibuffer_next_inst(); //Jin: handle cdp latency; @@ -901,18 +976,21 @@ void scheduler_unit::cycle() ready_inst = true; const active_mask_t &active_mask = m_simt_stack[warp_id]->get_active_mask(); assert( warp(warp_id).inst_in_pipeline() ); - if ( (pI->op == LOAD_OP)||(pI->op ==TENSOR_CORE_LOAD_OP)|| (pI->op == STORE_OP)|| (pI->op==TENSOR_CORE_STORE_OP) || (pI->op == MEMORY_BARRIER_OP) ) { - if( m_mem_out->has_free() ) { - m_shader->issue_warp(*m_mem_out,pI,active_mask,warp_id); - issued++; - issued_inst=true; - warp_inst_issued = true; - } + if ( (pI->op == LOAD_OP) || (pI->op == STORE_OP) || (pI->op == MEMORY_BARRIER_OP)||(pI->op==TENSOR_CORE_LOAD_OP)||(pI->op==TENSOR_CORE_STORE_OP) ) { + if( m_mem_out->has_free() && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::MEM)) { + m_shader->issue_warp(*m_mem_out,pI,active_mask,warp_id); + issued++; + issued_inst=true; + warp_inst_issued = true; + previous_issued_inst_exec_type = exec_unit_type_t::MEM; + } } else { + bool sp_pipe_avail = m_sp_out->has_free(); bool sfu_pipe_avail = m_sfu_out->has_free(); bool tensor_core_pipe_avail = m_tensor_core_out->has_free(); - if( sp_pipe_avail && (pI->op != SFU_OP) && (pI->op != TENSOR_CORE_OP) ) { + bool dp_pipe_avail = m_dp_out->has_free(); + if( sp_pipe_avail && (pI->op != TENSOR_CORE_OP) && (pI->op != SFU_OP && pI->op != DP_OP) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::SP)) { //Jin: special for CDP api if(pI->m_is_cdp && !warp(warp_id).m_cdp_dummy) { @@ -937,12 +1015,23 @@ void scheduler_unit::cycle() issued++; issued_inst=true; warp_inst_issued = true; - } else if ( (pI->op == SFU_OP) || (pI->op == ALU_SFU_OP) ) { + previous_issued_inst_exec_type = exec_unit_type_t::SP; + } else if ( (m_shader->m_config->gpgpu_num_dp_units != 0) && (pI->op == DP_OP) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::DP)) { + if( dp_pipe_avail ) { + m_shader->issue_warp(*m_dp_out,pI,active_mask,warp_id); + issued++; + issued_inst=true; + warp_inst_issued = true; + previous_issued_inst_exec_type = exec_unit_type_t::DP; + } + } //If the DP units = 0 (like in Fermi archi), then change DP inst to SFU inst + else if ( ((m_shader->m_config->gpgpu_num_dp_units == 0 && pI->op == DP_OP) || (pI->op == SFU_OP) || (pI->op == ALU_SFU_OP)) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::SFU)) { if( sfu_pipe_avail ) { m_shader->issue_warp(*m_sfu_out,pI,active_mask,warp_id); issued++; issued_inst=true; warp_inst_issued = true; + previous_issued_inst_exec_type = exec_unit_type_t::SFU; } } else if ( (pI->op == TENSOR_CORE_OP) ) { @@ -952,12 +1041,12 @@ void scheduler_unit::cycle() issued_inst=true; warp_inst_issued = true; } - } - } - } else { + } + }//end of else + } else { SCHED_DPRINTF( "Warp (warp_id %u, dynamic_warp_id %u) fails scoreboard\n", (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id() ); - } + } } } else if( valid ) { // this case can happen after a return instruction in diverged warp @@ -988,6 +1077,14 @@ void scheduler_unit::cycle() m_last_supervised_issued = supervised_iter; } } + + if(issued == 1) + m_stats->single_issue_nums[m_id]++; + else if(issued > 1) + m_stats->dual_issue_nums[m_id]++; + else + abort(); //issued should be > 0 + break; } } @@ -1045,6 +1142,16 @@ void gto_scheduler::order_warps() scheduler_unit::sort_warps_by_oldest_dynamic_id ); } +void oldest_scheduler::order_warps() +{ + order_by_priority( m_next_cycle_prioritized_warps, + m_supervised_warps, + m_last_supervised_issued, + m_supervised_warps.size(), + ORDERED_PRIORITY_FUNC_ONLY, + scheduler_unit::sort_warps_by_oldest_dynamic_id ); +} + void two_level_active_scheduler::do_on_warp_issued( unsigned warp_id, unsigned num_issued, @@ -1117,12 +1224,13 @@ swl_scheduler::swl_scheduler ( shader_core_stats* stats, shader_core_ctx* shader Scoreboard* scoreboard, simt_stack** simt, std::vector* warp, register_set* sp_out, + register_set* dp_out, register_set* sfu_out, register_set* tensor_core_out, register_set* mem_out, int id, char* config_string ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, tensor_core_out, mem_out, id ) + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out,tensor_core_out, mem_out, id ) { unsigned m_prioritization_readin; int ret = sscanf( config_string, @@ -1386,8 +1494,14 @@ ldst_unit::process_cache_access( cache_t* cache, mem_stage_stall_type result = NO_RC_FAIL; bool write_sent = was_write_sent(events); bool read_sent = was_read_sent(events); - if( write_sent ) - m_core->inc_store_req( inst.warp_id() ); + if( write_sent ) { + unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)? + (mf->get_data_size()/SECTOR_SIZE) : 1; + + for(unsigned i=0; i< inc_ack; ++i) + m_core->inc_store_req( inst.warp_id() ); + + } if ( status == HIT ) { assert( !read_sent ); inst.accessq_pop_back(); @@ -1399,7 +1513,7 @@ ldst_unit::process_cache_access( cache_t* cache, if( !write_sent ) delete mf; } else if ( status == RESERVATION_FAIL ) { - result = COAL_STALL; + result = BK_CONF; assert( !read_sent ); assert( !write_sent ); delete mf; @@ -1408,8 +1522,8 @@ ldst_unit::process_cache_access( cache_t* cache, //inst.clear_active( access.get_warp_mask() ); // threads in mf writeback when mf returns inst.accessq_pop_back(); } - if( !inst.accessq_empty() ) - result = BK_CONF; + if( !inst.accessq_empty() && result == NO_RC_FAIL) + result = COAL_STALL; return result; } @@ -1429,6 +1543,111 @@ mem_stage_stall_type ldst_unit::process_memory_access_queue( cache_t *cache, war return process_cache_access( cache, mf->get_addr(), inst, events, mf, status ); } +mem_stage_stall_type ldst_unit::process_memory_access_queue_l1cache( l1_cache *cache, warp_inst_t &inst ) +{ + mem_stage_stall_type result = NO_RC_FAIL; + if( inst.accessq_empty() ) + return result; + + mem_fetch *mf = m_mf_allocator->alloc(inst,inst.accessq_back()); + + if(m_config->m_L1D_config.l1_latency > 0) + { + if((l1_latency_queue[m_config->m_L1D_config.l1_latency-1]) == NULL) + { + l1_latency_queue[m_config->m_L1D_config.l1_latency-1] = mf; + + if( mf->get_inst().is_store() ) { + unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)? + (mf->get_data_size()/SECTOR_SIZE) : 1; + + for(unsigned i=0; i< inc_ack; ++i) + m_core->inc_store_req( inst.warp_id() ); + } + + inst.accessq_pop_back(); + } + else + { + result = BK_CONF; + delete mf; + } + if( !inst.accessq_empty() && result !=BK_CONF) + result = COAL_STALL; + return result; + } + else + { + std::list events; + enum cache_request_status status = cache->access(mf->get_addr(),mf,gpu_sim_cycle+gpu_tot_sim_cycle,events); + return process_cache_access( cache, mf->get_addr(), inst, events, mf, status ); + } +} + +void ldst_unit::L1_latency_queue_cycle() +{ + //std::deque< std::pair >::iterator it = m_latency_queue.begin(); + if((l1_latency_queue[0]) != NULL) + { + mem_fetch* mf_next = l1_latency_queue[0]; + std::list events; + enum cache_request_status status = m_L1D->access(mf_next->get_addr(),mf_next,gpu_sim_cycle+gpu_tot_sim_cycle,events); + + bool write_sent = was_write_sent(events); + bool read_sent = was_read_sent(events); + + if ( status == HIT ) { + assert( !read_sent ); + l1_latency_queue[0] = NULL; + if ( mf_next->get_inst().is_load() ) { + for ( unsigned r=0; r < 4; r++) + if (mf_next->get_inst().out[r] > 0) + { + assert(m_pending_writes[mf_next->get_inst().warp_id()][mf_next->get_inst().out[r]]>0); + unsigned still_pending = --m_pending_writes[mf_next->get_inst().warp_id()][mf_next->get_inst().out[r]]; + if(!still_pending) + { + m_pending_writes[mf_next->get_inst().warp_id()].erase(mf_next->get_inst().out[r]); + m_scoreboard->releaseRegister(mf_next->get_inst().warp_id(),mf_next->get_inst().out[r]); + m_core->warp_inst_complete(mf_next->get_inst()); + } + } + } + + //For write hit in WB policy + if(mf_next->get_inst().is_store() && !write_sent) + { + unsigned dec_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)? + (mf_next->get_data_size()/SECTOR_SIZE) : 1; + + mf_next->set_reply(); + + for(unsigned i=0; i< dec_ack; ++i) + m_core->store_ack(mf_next); + } + + if( !write_sent ) + delete mf_next; + + } else if ( status == RESERVATION_FAIL ) { + assert( !read_sent ); + assert( !write_sent ); + } else { + assert( status == MISS || status == HIT_RESERVED ); + l1_latency_queue[0] = NULL; + } + } + + for( unsigned stage = 0; stagem_L1D_config.l1_latency-1; ++stage) + if( l1_latency_queue[stage] == NULL) { + l1_latency_queue[stage] = l1_latency_queue[stage+1] ; + l1_latency_queue[stage+1] = NULL; + } + +} + + + bool ldst_unit::constant_cycle( warp_inst_t &inst, mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type) { if( inst.empty() || ((inst.space.get_type() != const_space) && (inst.space.get_type() != param_space_kernel)) ) @@ -1478,7 +1697,7 @@ bool ldst_unit::memory_cycle( warp_inst_t &inst, mem_stage_stall_type &stall_rea bypassL1D = true; } else if (inst.space.is_global()) { // global memory access // skip L1 cache if the option is enabled - if (m_core->get_config()->gmem_skip_L1D) + if (m_core->get_config()->gmem_skip_L1D && (CACHE_L1 != inst.cache_op)) bypassL1D = true; } if( bypassL1D ) { @@ -1502,9 +1721,9 @@ bool ldst_unit::memory_cycle( warp_inst_t &inst, mem_stage_stall_type &stall_rea } } else { assert( CACHE_UNDEFINED != inst.cache_op ); - stall_cond = process_memory_access_queue(m_L1D,inst); + stall_cond = process_memory_access_queue_l1cache(m_L1D,inst); } - if( !inst.accessq_empty() ) + if( !inst.accessq_empty() && stall_cond == NO_RC_FAIL) stall_cond = COAL_STALL; if (stall_cond != NO_RC_FAIL) { stall_reason = stall_cond; @@ -1534,6 +1753,11 @@ void ldst_unit::flush(){ m_L1D->flush(); } +void ldst_unit::invalidate(){ + // Flush L1D cache + m_L1D->invalidate(); +} + simd_function_unit::simd_function_unit( const shader_core_config *config ) { m_config=config; @@ -1586,6 +1810,13 @@ void sp_unit::active_lanes_in_pipeline(){ m_core->incfuactivelanes_stat(active_count); m_core->incfumemactivelanes_stat(active_count); } +void dp_unit::active_lanes_in_pipeline(){ + unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline(); + assert(active_count<=m_core->get_config()->warp_size); + m_core->incspactivelanes_stat(active_count); + m_core->incfuactivelanes_stat(active_count); + m_core->incfumemactivelanes_stat(active_count); +} void sfu::active_lanes_in_pipeline(){ unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline(); @@ -1610,6 +1841,12 @@ sp_unit::sp_unit( register_set* result_port, const shader_core_config *config,sh m_name = "SP "; } +dp_unit::dp_unit( register_set* result_port, const shader_core_config *config,shader_core_ctx *core) + : pipelined_simd_unit(result_port,config,config->max_sfu_latency,core) +{ + m_name = "DP "; +} + void sp_unit :: issue(register_set& source_reg) { warp_inst_t** ready_reg = source_reg.get_ready(); @@ -1619,6 +1856,14 @@ void sp_unit :: issue(register_set& source_reg) pipelined_simd_unit::issue(source_reg); } +void dp_unit :: issue(register_set& source_reg) +{ + warp_inst_t** ready_reg = source_reg.get_ready(); + //m_core->incexecstat((*ready_reg)); + (*ready_reg)->op_pipe=DP__OP; + m_core->incsp_stat(m_core->get_config()->warp_size,(*ready_reg)->latency); + pipelined_simd_unit::issue(source_reg); +} pipelined_simd_unit::pipelined_simd_unit( register_set* result_port, const shader_core_config *config, unsigned max_latency,shader_core_ctx *core ) : simd_function_unit(config) @@ -1712,8 +1957,9 @@ ldst_unit::ldst_unit( mem_fetch_interface *icnt, const memory_config *mem_config, shader_core_stats *stats, unsigned sid, - unsigned tpc ) : pipelined_simd_unit(NULL,config,3,core), m_next_wb(config) + unsigned tpc ) : pipelined_simd_unit(NULL,config,config->smem_latency,core), m_next_wb(config) { + assert(config->smem_latency > 1); init( icnt, mf_allocator, core, @@ -1734,6 +1980,12 @@ ldst_unit::ldst_unit( mem_fetch_interface *icnt, m_icnt, m_mf_allocator, IN_L1D_MISS_QUEUE ); + + if(m_config->m_L1D_config.l1_latency > 0) + { + for(int i=0; im_L1D_config.l1_latency; i++ ) + l1_latency_queue.push_back((mem_fetch*)NULL); + } } } @@ -1916,12 +2168,12 @@ void ldst_unit::cycle() if( !m_response_fifo.empty() ) { mem_fetch *mf = m_response_fifo.front(); - if (mf->istexture()) { + if (mf->get_access_type() == TEXTURE_ACC_R) { if (m_L1T->fill_port_free()) { m_L1T->fill(mf,gpu_sim_cycle+gpu_tot_sim_cycle); m_response_fifo.pop_front(); } - } else if (mf->isconst()) { + } else if (mf->get_access_type() == CONST_ACC_R) { if (m_L1C->fill_port_free()) { mf->set_status(IN_SHADER_FETCHED,gpu_sim_cycle+gpu_tot_sim_cycle); m_L1C->fill(mf,gpu_sim_cycle+gpu_tot_sim_cycle); @@ -1960,7 +2212,11 @@ void ldst_unit::cycle() m_L1T->cycle(); m_L1C->cycle(); - if( m_L1D ) m_L1D->cycle(); + if( m_L1D ) { + m_L1D->cycle(); + if(m_config->m_L1D_config.l1_latency > 0) + L1_latency_queue_cycle(); + } warp_inst_t &pipe_reg = *m_dispatch_reg; enum mem_stage_stall_type rc_fail = NO_RC_FAIL; @@ -1983,9 +2239,9 @@ void ldst_unit::cycle() unsigned warp_id = pipe_reg.warp_id(); if( pipe_reg.is_load() ) { if( pipe_reg.space.get_type() == shared_space ) { - if( m_pipeline_reg[2]->empty() ) { + if( m_pipeline_reg[m_config->smem_latency-1]->empty() ) { // new shared memory request - move_warp(m_pipeline_reg[2],m_dispatch_reg); + move_warp(m_pipeline_reg[m_config->smem_latency-1],m_dispatch_reg); m_dispatch_reg->clear(); } } else { @@ -2562,9 +2818,42 @@ unsigned int shader_core_config::max_cta( const kernel_info_t &k ) const assert( result <= MAX_CTA_PER_SHADER ); if (result < 1) { printf ("GPGPU-Sim uArch: ERROR ** Kernel requires more resources than shader has.\n"); + if(gpgpu_ignore_resources_limitation) { + printf ("GPGPU-Sim uArch: gpgpu_ignore_resources_limitation is set, ignore the ERROR!\n"); + return 1; + } abort(); } + if(adpative_volta_cache_config && !k.volta_cache_config_set) { + //For Volta, we assign the remaining shared memory to L1 cache + //For more info, see https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#shared-memory-7-x + unsigned total_shmed = kernel_info->smem * result; + assert(total_shmed >=0 && total_shmed <= gpgpu_shmem_size); + assert(gpgpu_shmem_size == 98304); //Volta has 96 KB shared + assert(m_L1D_config.get_nset() == 4); //Volta L1 has four sets + if(total_shmed < gpgpu_shmem_size){ + if(total_shmed == 0) + m_L1D_config.set_assoc(256); //L1 is 128KB ans shd=0 + else if(total_shmed > 0 && total_shmed <= 8192) + m_L1D_config.set_assoc(240); //L1 is 120KB ans shd=8KB + else if(total_shmed > 8192 && total_shmed <= 16384) + m_L1D_config.set_assoc(224); //L1 is 112KB ans shd=16KB + else if(total_shmed > 16384 && total_shmed <= 32768) + m_L1D_config.set_assoc(192); //L1 is 96KB ans shd=32KB + else if(total_shmed > 32768 && total_shmed <= 65536) + m_L1D_config.set_assoc(128); //L1 is 64KB ans shd=64KB + else if(total_shmed > 65536 && total_shmed <= gpgpu_shmem_size) + m_L1D_config.set_assoc(64); //L1 is 32KB and shd=96KB + else + assert(0); + + printf ("GPGPU-Sim: Reconfigure L1 cache in Volta Archi to %uKB\n", m_L1D_config.get_total_size_inKB()); + } + + k.volta_cache_config_set = true; + } + return result; } @@ -2586,6 +2875,11 @@ void shader_core_ctx::cache_flush() m_ldst_unit->flush(); } +void shader_core_ctx::cache_invalidate() +{ + m_ldst_unit->invalidate(); +} + // modifiers std::list opndcoll_rfu_t::arbiter_t::allocate_reads() { @@ -3340,6 +3634,15 @@ void simt_core_cluster::print_not_completed( FILE *fp ) const } } + +float simt_core_cluster::get_current_occupancy( unsigned long long& active, unsigned long long& total ) const { + float aggregate = 0.f; + for( unsigned i=0; i < m_config->n_simt_cores_per_cluster; i++ ) { + aggregate+=m_core[i]->get_current_occupancy( active, total ); + } + return aggregate / m_config->n_simt_cores_per_cluster; +} + unsigned simt_core_cluster::get_n_active_cta() const { unsigned n=0; @@ -3403,6 +3706,12 @@ void simt_core_cluster::cache_flush() m_core[i]->cache_flush(); } +void simt_core_cluster::cache_invalidate() +{ + for( unsigned i=0; i < m_config->n_simt_cores_per_cluster; i++ ) + m_core[i]->cache_invalidate(); +} + bool simt_core_cluster::icnt_injection_buffer_full(unsigned size, bool write) { unsigned request_size = size; diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index 97e438f..437506c 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -70,6 +70,14 @@ #define WRITE_MASK_SIZE 8 +enum exec_unit_type_t +{ + NONE = 0, + SP = 1, + SFU = 2, + MEM = 3, + DP = 4 +}; class thread_ctx_t { public: @@ -308,6 +316,7 @@ enum concrete_scheduler CONCRETE_SCHEDULER_GTO, CONCRETE_SCHEDULER_TWO_LEVEL_ACTIVE, CONCRETE_SCHEDULER_WARP_LIMITING, + CONCRETE_SCHEDULER_OLDEST_FIRST, NUM_CONCRETE_SCHEDULERS }; @@ -317,13 +326,14 @@ public: Scoreboard* scoreboard, simt_stack** simt, std::vector* warp, register_set* sp_out, + register_set* dp_out, register_set* sfu_out, register_set* tensor_core_out, register_set* mem_out, int id) : m_supervised_warps(), m_stats(stats), m_shader(shader), m_scoreboard(scoreboard), m_simt_stack(simt), /*m_pipeline_reg(pipe_regs),*/ m_warp(warp), - m_sp_out(sp_out),m_sfu_out(sfu_out),m_tensor_core_out(tensor_core_out),m_mem_out(mem_out), m_id(id){} + m_sp_out(sp_out),m_dp_out(dp_out),m_sfu_out(sfu_out),m_tensor_core_out(tensor_core_out),m_mem_out(mem_out), m_id(id){} virtual ~scheduler_unit(){} virtual void add_supervised_warp_id(int i) { m_supervised_warps.push_back(&warp(i)); @@ -395,6 +405,7 @@ protected: //warp_inst_t** m_pipeline_reg; std::vector* m_warp; register_set* m_sp_out; + register_set* m_dp_out; register_set* m_sfu_out; register_set* m_tensor_core_out; register_set* m_mem_out; @@ -408,11 +419,12 @@ public: Scoreboard* scoreboard, simt_stack** simt, std::vector* warp, register_set* sp_out, + register_set* dp_out, register_set* sfu_out, register_set* tensor_core_out, register_set* mem_out, int id ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, tensor_core_out, mem_out, id ){} + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out,tensor_core_out, mem_out, id ){} virtual ~lrr_scheduler () {} virtual void order_warps (); virtual void done_adding_supervised_warps() { @@ -426,11 +438,12 @@ public: Scoreboard* scoreboard, simt_stack** simt, std::vector* warp, register_set* sp_out, + register_set* dp_out, register_set* sfu_out, register_set* tensor_core_out, register_set* mem_out, int id ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, tensor_core_out, mem_out, id ){} + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out,tensor_core_out, mem_out, id ){} virtual ~gto_scheduler () {} virtual void order_warps (); virtual void done_adding_supervised_warps() { @@ -439,6 +452,25 @@ public: }; +class oldest_scheduler : public scheduler_unit { +public: + oldest_scheduler ( shader_core_stats* stats, shader_core_ctx* shader, + Scoreboard* scoreboard, simt_stack** simt, + std::vector* warp, + register_set* sp_out, + register_set* dp_out, + register_set* sfu_out, + register_set* tensor_core_out, + register_set* mem_out, + int id ) + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out,tensor_core_out, mem_out, id ){} + virtual ~oldest_scheduler () {} + virtual void order_warps (); + virtual void done_adding_supervised_warps() { + m_last_supervised_issued = m_supervised_warps.begin(); + } + +}; class two_level_active_scheduler : public scheduler_unit { public: @@ -446,12 +478,13 @@ public: Scoreboard* scoreboard, simt_stack** simt, std::vector* warp, register_set* sp_out, + register_set* dp_out, register_set* sfu_out, register_set* tensor_core_out, register_set* mem_out, int id, char* config_str ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, tensor_core_out, mem_out, id ), + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out,tensor_core_out, mem_out, id ), m_pending_warps() { unsigned inner_level_readin; @@ -497,6 +530,7 @@ public: Scoreboard* scoreboard, simt_stack** simt, std::vector* warp, register_set* sp_out, + register_set* dp_out, register_set* sfu_out, register_set* tensor_core_out, register_set* mem_out, @@ -1060,6 +1094,23 @@ public: switch(inst.op) { case SFU_OP: break; case ALU_SFU_OP: break; + case DP_OP: break; //for compute <= 29 (i..e Fermi and GT200) + default: return false; + } + return pipelined_simd_unit::can_issue(inst); + } + virtual void active_lanes_in_pipeline(); + virtual void issue( register_set& source_reg ); +}; + +class dp_unit : public pipelined_simd_unit +{ +public: + dp_unit( register_set* result_port, const shader_core_config *config, shader_core_ctx *core ); + virtual bool can_issue( const warp_inst_t &inst ) const + { + switch(inst.op) { + case DP_OP: break; default: return false; } return pipelined_simd_unit::can_issue(inst); @@ -1098,6 +1149,7 @@ public: case STORE_OP: return false; case TENSOR_CORE_STORE_OP: return false; case MEMORY_BARRIER_OP: return false; + case DP_OP: return false; default: break; } return pipelined_simd_unit::can_issue(inst); @@ -1129,6 +1181,7 @@ public: void fill( mem_fetch *mf ); void flush(); + void invalidate(); void writeback(); // accessors @@ -1195,6 +1248,7 @@ protected: mem_fetch *mf, enum cache_request_status status ); mem_stage_stall_type process_memory_access_queue( cache_t *cache, warp_inst_t &inst ); + mem_stage_stall_type process_memory_access_queue_l1cache( l1_cache *cache, warp_inst_t &inst ); const memory_config *m_memory_config; class mem_fetch_interface *m_icnt; @@ -1223,13 +1277,18 @@ protected: // for debugging unsigned long long m_last_inst_gpu_sim_cycle; unsigned long long m_last_inst_gpu_tot_sim_cycle; + + std::deque l1_latency_queue; + void L1_latency_queue_cycle(); }; enum pipeline_stage_name_t { ID_OC_SP=0, + ID_OC_DP, ID_OC_SFU, ID_OC_MEM, OC_EX_SP, + OC_EX_DP, OC_EX_SFU, OC_EX_MEM, EX_WB, @@ -1240,9 +1299,11 @@ enum pipeline_stage_name_t { const char* const pipeline_stage_name_decode[] = { "ID_OC_SP", + "ID_OC_DP", "ID_OC_SFU", "ID_OC_MEM", "OC_EX_SP", + "OC_EX_DP", "OC_EX_SFU", "OC_EX_MEM", "EX_WB", @@ -1340,27 +1401,29 @@ struct shader_core_config : public core_config mutable cache_config m_L1C_config; mutable l1d_cache_config m_L1D_config; - bool gmem_skip_L1D; // on = global memory access always skip the L1 cache - bool gpgpu_dwf_reg_bankconflict; int gpgpu_num_sched_per_core; int gpgpu_max_insn_issue_per_warp; + bool gpgpu_dual_issue_diff_exec_units; //op collector int gpgpu_operand_collector_num_units_sp; + int gpgpu_operand_collector_num_units_dp; int gpgpu_operand_collector_num_units_sfu; int gpgpu_operand_collector_num_units_tensor_core; int gpgpu_operand_collector_num_units_mem; int gpgpu_operand_collector_num_units_gen; unsigned int gpgpu_operand_collector_num_in_ports_sp; + unsigned int gpgpu_operand_collector_num_in_ports_dp; unsigned int gpgpu_operand_collector_num_in_ports_sfu; unsigned int gpgpu_operand_collector_num_in_ports_tensor_core; unsigned int gpgpu_operand_collector_num_in_ports_mem; unsigned int gpgpu_operand_collector_num_in_ports_gen; unsigned int gpgpu_operand_collector_num_out_ports_sp; + unsigned int gpgpu_operand_collector_num_out_ports_dp; unsigned int gpgpu_operand_collector_num_out_ports_sfu; unsigned int gpgpu_operand_collector_num_out_ports_tensor_core; unsigned int gpgpu_operand_collector_num_out_ports_mem; @@ -1368,6 +1431,7 @@ struct shader_core_config : public core_config int gpgpu_num_sp_units; int gpgpu_tensor_core_avail; + int gpgpu_num_dp_units; int gpgpu_num_sfu_units; int gpgpu_num_tensor_core_units; int gpgpu_num_mem_units; @@ -1379,6 +1443,7 @@ struct shader_core_config : public core_config unsigned gpgpu_num_reg_banks; bool gpgpu_reg_bank_use_warp_id; bool gpgpu_local_mem_map; + bool gpgpu_ignore_resources_limitation; unsigned max_sp_latency; unsigned max_sfu_latency; @@ -1391,10 +1456,14 @@ struct shader_core_config : public core_config int simt_core_sim_order; + unsigned smem_latency; + unsigned mem2device(unsigned memid) const { return memid + n_simt_clusters; } //Jin: concurrent kernel on sm bool gpgpu_concurrent_kernel_sm; + + bool adpative_volta_cache_config; }; struct shader_core_stats_pod { @@ -1457,6 +1526,8 @@ struct shader_core_stats_pod { unsigned *last_shader_cycle_distro; unsigned *num_warps_issuable; unsigned gpgpu_n_stall_shd_mem; + unsigned* single_issue_nums; + unsigned* dual_issue_nums; //memory access classification int gpgpu_n_mem_read_local; @@ -1528,6 +1599,8 @@ public: m_n_diverge = (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); shader_cycle_distro = (unsigned*) calloc(config->warp_size+3, sizeof(unsigned)); last_shader_cycle_distro = (unsigned*) calloc(m_config->warp_size+3, sizeof(unsigned)); + single_issue_nums = (unsigned*) calloc(config->gpgpu_num_sched_per_core,sizeof(unsigned)); + dual_issue_nums = (unsigned*) calloc(config->gpgpu_num_sched_per_core, sizeof(unsigned)); n_simt_to_mem = (long *)calloc(config->num_shader(), sizeof(long)); n_mem_to_simt = (long *)calloc(config->num_shader(), sizeof(long)); @@ -1651,6 +1724,7 @@ public: void issue_block2core( class kernel_info_t &kernel ); void cache_flush(); + void cache_invalidate(); void accept_fetch_response( mem_fetch *mf ); void accept_ldst_unit_response( class mem_fetch * mf ); void broadcast_barrier_reduction(unsigned cta_id, unsigned bar_id,warp_set_t warps); @@ -1679,6 +1753,7 @@ public: // accessors virtual bool warp_waiting_at_barrier( unsigned warp_id ) const; void get_pdom_stack_top_info( unsigned tid, unsigned *pc, unsigned *rpc ) const; + float get_current_occupancy( unsigned long long & active, unsigned long long & total ) const; // used by pipeline timing model components: // modifiers @@ -1885,10 +1960,14 @@ public: std::vector m_pipeline_reg; Scoreboard *m_scoreboard; opndcoll_rfu_t m_operand_collector; + int m_active_warps; //schedule std::vector schedulers; + //issue + unsigned int Issue_Prio; + // execute unsigned m_num_function_units; std::vector m_dispatch_port; @@ -1940,6 +2019,7 @@ public: void reinit(); unsigned issue_block2core(); void cache_flush(); + void cache_invalidate(); bool icnt_injection_buffer_full(unsigned size, bool write); void icnt_inject_request_packet(class mem_fetch *mf); @@ -1970,6 +2050,7 @@ public: void get_L1T_sub_stats(struct cache_sub_stats &css) const; void get_icnt_stats(long &n_simt_to_mem, long &n_mem_to_simt) const; + float get_current_occupancy( unsigned long long& active, unsigned long long & total ) const; private: unsigned m_cluster_id; diff --git a/src/gpgpusim_entrypoint.cc b/src/gpgpusim_entrypoint.cc index ede9f20..52e2f5e 100644 --- a/src/gpgpusim_entrypoint.cc +++ b/src/gpgpusim_entrypoint.cc @@ -172,6 +172,7 @@ void *gpgpu_sim_thread_concurrent(void*) g_sim_active = false; pthread_mutex_unlock(&g_sim_lock); } while( !g_sim_done ); + printf("GPGPU-Sim: *** simulation thread exiting ***\n"); fflush(stdout); diff --git a/src/gpuwattch/makefile b/src/gpuwattch/makefile index ab718cc..354c9ec 100644 --- a/src/gpuwattch/makefile +++ b/src/gpuwattch/makefile @@ -11,10 +11,10 @@ opt: $(TAR).mk obj_opt @$(MAKE) TAG=opt -C . -f $(TAR).mk obj_dbg: - mkdir $@ + mkdir -p $@ obj_opt: - mkdir $@ + mkdir -p $@ depend: @$(MAKE) TAG=opt -C . -f $(TAR).mk depend diff --git a/src/intersim2/Makefile b/src/intersim2/Makefile index bd42000..7d10b3f 100644 --- a/src/intersim2/Makefile +++ b/src/intersim2/Makefile @@ -36,7 +36,7 @@ DEBUG ?= 0 LEX = flex YACC = bison -y DEFINE = #-DTRACK_STALLS -DTRACK_BUFFERS -DTRACK_FLOWS -DTRACK_CREDITS -INCPATH = -I. -Iarbiters -Iallocators -Irouters -Inetworks -Ipower +INCPATH = -I. -Iarbiters -Iallocators -Irouters -Inetworks -Ipower -I$(GPGPUSIM_ROOT)/src ifeq ($(CREATE_LIBRARY),1) INCPATH += -I$(GPGPUSIM_ROOT)/src/gpgpu-sim/ diff --git a/src/intersim2/interconnect_interface.cpp b/src/intersim2/interconnect_interface.cpp index 4386821..1e1a2d7 100644 --- a/src/intersim2/interconnect_interface.cpp +++ b/src/intersim2/interconnect_interface.cpp @@ -44,6 +44,7 @@ #include "booksim.hpp" #include "intersim_config.hpp" #include "network.hpp" +#include "trace.h" InterconnectInterface* InterconnectInterface::New(const char* const config_file) { @@ -147,6 +148,8 @@ void InterconnectInterface::Push(unsigned input_deviceID, unsigned output_device { // it should have free buffer assert(HasBuffer(input_deviceID, size)); + + DPRINTF(INTERCONNECT, "Sent %d bytes from %d to %d", size, input_deviceID, output_deviceID); int output_icntID = _node_map[output_deviceID]; int input_icntID = _node_map[input_deviceID]; @@ -178,7 +181,11 @@ void InterconnectInterface::Push(unsigned input_deviceID, unsigned output_device case WRITE_REQUEST: packet_type = Flit::WRITE_REQUEST ;break; case READ_REPLY: packet_type = Flit::READ_REPLY ;break; case WRITE_ACK: packet_type = Flit::WRITE_REPLY ;break; - default: assert (0); + default: + { + cout<<"Type "<get_type()<<" is undefined!"< Date: Sun, 7 Jul 2019 23:56:21 -0400 Subject: g_keep_intermediate_files Signed-off-by: Mengchi Zhang --- libcuda/gpgpu_context.h | 1 + libopencl/opencl_runtime_api.cc | 2 +- src/cuda-sim/ptx_ir.h | 3 --- src/cuda-sim/ptx_loader.cc | 7 +++---- src/cuda-sim/ptx_loader.h | 3 ++- 5 files changed, 7 insertions(+), 9 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/libcuda/gpgpu_context.h b/libcuda/gpgpu_context.h index d819559..a8b60f4 100644 --- a/libcuda/gpgpu_context.h +++ b/libcuda/gpgpu_context.h @@ -32,6 +32,7 @@ class gpgpu_context { class symbol_table* init_parser(const char*); class gpgpu_sim *gpgpu_ptx_sim_init_perf(); struct _cuda_device_id *GPGPUSim_Init(); + void ptx_reg_options(option_parser_t opp); }; gpgpu_context* GPGPU_Context(); diff --git a/libopencl/opencl_runtime_api.cc b/libopencl/opencl_runtime_api.cc index e91e2e0..03ec80c 100644 --- a/libopencl/opencl_runtime_api.cc +++ b/libopencl/opencl_runtime_api.cc @@ -537,7 +537,7 @@ void _cl_program::Build(const char *options) exit(1); } } - if( !g_keep_intermediate_files ) { + if( !ctx->ptxinfo->g_keep_intermediate_files ) { // clean up files... snprintf(commandline,1024,"rm -f %s", cl_fname ); int result = system(commandline); diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 1af85de..fd869c6 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1575,11 +1575,8 @@ struct textureInfo { extern std::map g_sym_name_to_symbol_table; -extern bool g_keep_intermediate_files; - void gpgpu_ptx_assemble( std::string kname, void *kinfo ); #include "../option_parser.h" -void ptx_reg_options(option_parser_t opp); unsigned ptx_kernel_shmem_size( void *kernel_impl ); unsigned ptx_kernel_nregs( void *kernel_impl ); diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc index 921d1e6..e0d9a11 100644 --- a/src/cuda-sim/ptx_loader.cc +++ b/src/cuda-sim/ptx_loader.cc @@ -58,17 +58,16 @@ extern int ptxinfo_lex_destroy(yyscan_t scanner); static bool g_save_embedded_ptx; static int g_occupancy_sm_number; -bool g_keep_intermediate_files; bool m_ptx_save_converted_ptxplus; -bool keep_intermediate_files() {return g_keep_intermediate_files;} +bool ptxinfo_data::keep_intermediate_files() {return g_keep_intermediate_files;} -void ptx_reg_options(option_parser_t opp) +void gpgpu_context::ptx_reg_options(option_parser_t opp) { option_parser_register(opp, "-save_embedded_ptx", OPT_BOOL, &g_save_embedded_ptx, "saves ptx files embedded in binary as .ptx", "0"); - option_parser_register(opp, "-keep", OPT_BOOL, &g_keep_intermediate_files, + option_parser_register(opp, "-keep", OPT_BOOL, &(ptxinfo->g_keep_intermediate_files), "keep intermediate files created by GPGPU-Sim when interfacing with external programs", "0"); option_parser_register(opp, "-gpgpu_ptx_save_converted_ptxplus", OPT_BOOL, diff --git a/src/cuda-sim/ptx_loader.h b/src/cuda-sim/ptx_loader.h index c214b95..aa3e2f3 100644 --- a/src/cuda-sim/ptx_loader.h +++ b/src/cuda-sim/ptx_loader.h @@ -42,7 +42,9 @@ class ptxinfo_data{ unsigned col; const char *g_ptxinfo_filename; class gpgpu_context* gpgpu_ctx; + bool g_keep_intermediate_files; void ptxinfo_addinfo(); + bool keep_intermediate_files(); }; @@ -50,6 +52,5 @@ extern bool g_override_embedded_ptx; extern int no_of_ptx; //counter to track number of ptx files to be extracted in an application. char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptx_str, const std::string sass_str, const std::string elf_str); -bool keep_intermediate_files(); #endif -- cgit v1.3 From b3e786e3d8d720217f36a214e9b5be9a19ab9dd2 Mon Sep 17 00:00:00 2001 From: Mengchi Zhang Date: Mon, 8 Jul 2019 12:37:52 -0400 Subject: Move opcode_latency_int thus pass gpgpu_context into many classes Signed-off-by: Mengchi Zhang --- libcuda/gpgpu_context.h | 3 +++ src/cuda-sim/cuda-sim.cc | 7 ++++--- src/cuda-sim/cuda-sim.h | 11 +++++++++-- src/cuda-sim/ptx_ir.cc | 4 +++- src/cuda-sim/ptx_ir.h | 7 ++++++- src/cuda-sim/ptx_parser.cc | 3 ++- src/gpgpu-sim/gpu-sim.cc | 2 +- src/gpgpu-sim/gpu-sim.h | 11 ++++++++--- src/gpgpu-sim/mem_latency_stat.cc | 2 +- src/gpgpu-sim/mem_latency_stat.h | 4 ++-- src/gpgpu-sim/power_interface.cc | 2 +- src/gpgpu-sim/power_interface.h | 2 +- src/gpgpu-sim/power_stat.cc | 6 +++--- src/gpgpu-sim/power_stat.h | 6 +++--- src/gpgpu-sim/shader.cc | 7 ++++--- src/gpgpu-sim/shader.h | 18 ++++++++++++------ src/gpgpusim_entrypoint.cc | 4 ++-- 17 files changed, 65 insertions(+), 34 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/libcuda/gpgpu_context.h b/libcuda/gpgpu_context.h index a8b60f4..2e21009 100644 --- a/libcuda/gpgpu_context.h +++ b/libcuda/gpgpu_context.h @@ -4,6 +4,7 @@ #include "../src/cuda-sim/ptx_loader.h" #include "../src/cuda-sim/ptx_parser.h" #include "../src/gpgpusim_entrypoint.h" +#include "../src/cuda-sim/cuda-sim.h" class gpgpu_context { public: @@ -13,6 +14,7 @@ class gpgpu_context { ptxinfo = new ptxinfo_data(this); ptx_parser = new ptx_recognizer(this); the_gpgpusim = new GPGPUsim_ctx(this); + func_sim = new cuda_sim(); } // global list symbol_table *g_global_allfiles_symbol_table; @@ -22,6 +24,7 @@ class gpgpu_context { ptxinfo_data* ptxinfo; ptx_recognizer* ptx_parser; GPGPUsim_ctx* the_gpgpusim; + cuda_sim* func_sim; // member function list void cuobjdumpParseBinary(unsigned int handle); class symbol_table *gpgpu_ptx_sim_load_ptx_from_string( const char *p, unsigned source_num ); diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 261d605..df0bbd7 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -51,6 +51,7 @@ typedef void * yyscan_t; #include "decuda_pred_table/decuda_pred_table.h" #include "../stream_manager.h" #include "cuda_device_runtime.h" +#include "../../libcuda/gpgpu_context.h" int gpgpu_ptx_instruction_classification; void ** g_inst_classification_stat = NULL; @@ -66,12 +67,12 @@ int cp_cta_resume; unsigned g_ptx_sim_num_insn = 0; unsigned gpgpu_param_num_shaders = 0; -char *opcode_latency_int, *opcode_latency_fp, *opcode_latency_dp,*opcode_latency_sfu,*opcode_latency_tensor; +char *opcode_latency_fp, *opcode_latency_dp,*opcode_latency_sfu,*opcode_latency_tensor; char *opcode_initiation_int, *opcode_initiation_fp, *opcode_initiation_dp,*opcode_initiation_sfu,*opcode_initiation_tensor; char *cdp_latency_str; unsigned cdp_latency[5]; -void ptx_opcocde_latency_options (option_parser_t opp) { +void cuda_sim::ptx_opcocde_latency_options (option_parser_t opp) { option_parser_register(opp, "-ptx_opcode_latency_int", OPT_CSTR, &opcode_latency_int, "Opcode latencies for integers " "Default 1,1,19,25,145", @@ -667,7 +668,7 @@ void ptx_instruction::set_opcode_and_latency() * [3] MAD * [4] DIV */ - sscanf(opcode_latency_int, "%u,%u,%u,%u,%u", + sscanf(gpgpu_ctx->func_sim->opcode_latency_int, "%u,%u,%u,%u,%u", &int_latency[0],&int_latency[1],&int_latency[2], &int_latency[3],&int_latency[4]); sscanf(opcode_latency_fp, "%u,%u,%u,%u,%u", diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h index e690356..96d34f5 100644 --- a/src/cuda-sim/cuda-sim.h +++ b/src/cuda-sim/cuda-sim.h @@ -47,10 +47,9 @@ extern int g_debug_thread_uid; extern void ** g_inst_classification_stat; extern void ** g_inst_op_classification_stat; extern int g_ptx_kernel_count; // used for classification stat collection purposes -extern char *opcode_latency_int, *opcode_latency_fp, *opcode_latency_dp,*opcode_latency_sfu,*opcode_latency_tensor; +extern char *opcode_latency_fp, *opcode_latency_dp,*opcode_latency_sfu,*opcode_latency_tensor; -void ptx_opcocde_latency_options (option_parser_t opp); extern class kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, gpgpu_ptx_sim_arg_list_t args, struct dim3 gridDim, @@ -134,4 +133,12 @@ void print_ptxinfo(); void clear_ptxinfo(); struct gpgpu_ptx_sim_info get_ptxinfo(); +class cuda_sim { + public: + //global variables + char *opcode_latency_int; + //global functions + void ptx_opcocde_latency_options (option_parser_t opp); +}; + #endif diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index 8cedf79..1bd409e 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -1095,8 +1095,10 @@ ptx_instruction::ptx_instruction( int opcode, const char *file, unsigned line, const char *source, - const core_config *config ) : warp_inst_t(config) + const core_config *config, + gpgpu_context* ctx ) : warp_inst_t(config) { + gpgpu_ctx = ctx; m_uid = ++g_num_ptx_inst_uid; m_PC = 0; m_opcode = opcode; diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index fd869c6..1604551 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -43,6 +43,8 @@ #include "memory.h" +class gpgpu_context; + class type_info_key { public: type_info_key() @@ -931,7 +933,8 @@ public: const char *file, unsigned line, const char *source, - const core_config *config ); + const core_config *config, + gpgpu_context* ctx); void print_insn() const; virtual void print_insn( FILE *fp ) const; @@ -1187,6 +1190,8 @@ private: virtual void pre_decode(); friend class function_info; static unsigned g_num_ptx_inst_uid; + // backward pointer + class gpgpu_context* gpgpu_ctx; }; class param_info { diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index 05fc618..0139534 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -285,7 +285,8 @@ void ptx_recognizer::add_instruction() gpgpu_ctx->g_filename, ptx_get_lineno(scanner), linebuf, - g_shader_core_config ); + g_shader_core_config, + gpgpu_ctx ); g_instructions.push_back(i); g_inst_lookup[gpgpu_ctx->g_filename][ptx_get_lineno(scanner)] = i; init_instruction_state(); diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 2ff37d1..39acdd9 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -1826,7 +1826,7 @@ void gpgpu_sim::dump_pipeline( int mask, int s, int m ) const fflush(stdout); } -const struct shader_core_config * gpgpu_sim::getShaderCoreConfig() +const shader_core_config * gpgpu_sim::getShaderCoreConfig() { return m_shader_config; } diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h index e2c913a..7eeb7dd 100644 --- a/src/gpgpu-sim/gpu-sim.h +++ b/src/gpgpu-sim/gpu-sim.h @@ -295,7 +295,10 @@ extern bool g_interactive_debugger_enabled; class gpgpu_sim_config : public power_config, public gpgpu_functional_sim_config { public: - gpgpu_sim_config() { m_valid = false; } + gpgpu_sim_config(gpgpu_context* ctx): m_shader_config(ctx) { + m_valid = false; + gpgpu_ctx = ctx; + } void reg_options(class OptionParser * opp); void init() { @@ -341,6 +344,8 @@ private: void init_clock_domains(void ); + // backward pointer + class gpgpu_context* gpgpu_ctx; bool m_valid; shader_core_config m_shader_config; memory_config m_memory_config; @@ -473,7 +478,7 @@ public: /*! * Returning the configuration of the shader core, used by the functional simulation only so far */ - const struct shader_core_config * getShaderCoreConfig(); + const shader_core_config * getShaderCoreConfig(); //! Get shader core Memory Configuration @@ -537,7 +542,7 @@ private: const gpgpu_sim_config &m_config; const struct cudaDeviceProp *m_cuda_properties; - const struct shader_core_config *m_shader_config; + const shader_core_config *m_shader_config; const struct memory_config *m_memory_config; // stats diff --git a/src/gpgpu-sim/mem_latency_stat.cc b/src/gpgpu-sim/mem_latency_stat.cc index 04dc75b..d08ba39 100644 --- a/src/gpgpu-sim/mem_latency_stat.cc +++ b/src/gpgpu-sim/mem_latency_stat.cc @@ -42,7 +42,7 @@ #include #include -memory_stats_t::memory_stats_t( unsigned n_shader, const struct shader_core_config *shader_config, const struct memory_config *mem_config, const class gpgpu_sim* gpu ) +memory_stats_t::memory_stats_t( unsigned n_shader, const shader_core_config *shader_config, const struct memory_config *mem_config, const class gpgpu_sim* gpu ) { assert( mem_config->m_valid ); assert( shader_config->m_valid ); diff --git a/src/gpgpu-sim/mem_latency_stat.h b/src/gpgpu-sim/mem_latency_stat.h index b9285c1..6ce568d 100644 --- a/src/gpgpu-sim/mem_latency_stat.h +++ b/src/gpgpu-sim/mem_latency_stat.h @@ -35,7 +35,7 @@ class memory_stats_t { public: memory_stats_t( unsigned n_shader, - const struct shader_core_config *shader_config, + const class shader_core_config *shader_config, const struct memory_config *mem_config, const class gpgpu_sim* gpu); @@ -53,7 +53,7 @@ public: unsigned m_n_shader; - const struct shader_core_config *m_shader_config; + const shader_core_config *m_shader_config; const struct memory_config *m_memory_config; const class gpgpu_sim* m_gpu; diff --git a/src/gpgpu-sim/power_interface.cc b/src/gpgpu-sim/power_interface.cc index 3861b6a..0272aa6 100644 --- a/src/gpgpu-sim/power_interface.cc +++ b/src/gpgpu-sim/power_interface.cc @@ -38,7 +38,7 @@ void init_mcpat(const gpgpu_sim_config &config, class gpgpu_sim_wrapper *wrapper } -void mcpat_cycle(const gpgpu_sim_config &config, const struct shader_core_config *shdr_config, class gpgpu_sim_wrapper *wrapper, class power_stat_t *power_stats, unsigned stat_sample_freq, unsigned tot_cycle, unsigned cycle, unsigned tot_inst, unsigned inst){ +void mcpat_cycle(const gpgpu_sim_config &config, const shader_core_config *shdr_config, class gpgpu_sim_wrapper *wrapper, class power_stat_t *power_stats, unsigned stat_sample_freq, unsigned tot_cycle, unsigned cycle, unsigned tot_inst, unsigned inst){ static bool mcpat_init=true; diff --git a/src/gpgpu-sim/power_interface.h b/src/gpgpu-sim/power_interface.h index afac22b..a388c23 100644 --- a/src/gpgpu-sim/power_interface.h +++ b/src/gpgpu-sim/power_interface.h @@ -36,7 +36,7 @@ #include "gpgpu_sim_wrapper.h" void init_mcpat(const gpgpu_sim_config &config, class gpgpu_sim_wrapper *wrapper, unsigned stat_sample_freq, unsigned tot_inst, unsigned inst); -void mcpat_cycle(const gpgpu_sim_config &config, const struct shader_core_config *shdr_config, class gpgpu_sim_wrapper *wrapper, class power_stat_t *power_stats, +void mcpat_cycle(const gpgpu_sim_config &config, const shader_core_config *shdr_config, class gpgpu_sim_wrapper *wrapper, class power_stat_t *power_stats, unsigned stat_sample_freq, unsigned tot_cycle, unsigned cycle, unsigned tot_inst, unsigned inst); void mcpat_reset_perf_count(class gpgpu_sim_wrapper *wrapper); diff --git a/src/gpgpu-sim/power_stat.cc b/src/gpgpu-sim/power_stat.cc index 4c995e9..007b4c6 100644 --- a/src/gpgpu-sim/power_stat.cc +++ b/src/gpgpu-sim/power_stat.cc @@ -42,7 +42,7 @@ -power_mem_stat_t::power_mem_stat_t(const struct memory_config *mem_config, const struct shader_core_config *shdr_config, memory_stats_t *mem_stats, shader_core_stats *shdr_stats){ +power_mem_stat_t::power_mem_stat_t(const struct memory_config *mem_config, const shader_core_config *shdr_config, memory_stats_t *mem_stats, shader_core_stats *shdr_stats){ assert( mem_config->m_valid ); m_mem_stats = mem_stats; m_config = mem_config; @@ -125,7 +125,7 @@ void power_mem_stat_t::print (FILE *fout) const { } -power_core_stat_t::power_core_stat_t( const struct shader_core_config *shader_config, shader_core_stats *core_stats ) +power_core_stat_t::power_core_stat_t( const shader_core_config *shader_config, shader_core_stats *core_stats ) { assert( shader_config->m_valid ); m_config = shader_config; @@ -266,7 +266,7 @@ for(unsigned i=0; inum_shader(); ++i){ } } -power_stat_t::power_stat_t( const struct shader_core_config *shader_config,float * average_pipeline_duty_cycle,float *active_sms,shader_core_stats * shader_stats, const struct memory_config *mem_config,memory_stats_t * memory_stats) +power_stat_t::power_stat_t( const shader_core_config *shader_config,float * average_pipeline_duty_cycle,float *active_sms,shader_core_stats * shader_stats, const struct memory_config *mem_config,memory_stats_t * memory_stats) { assert( shader_config->m_valid ); assert( mem_config->m_valid ); diff --git a/src/gpgpu-sim/power_stat.h b/src/gpgpu-sim/power_stat.h index 20af2e5..91fade9 100644 --- a/src/gpgpu-sim/power_stat.h +++ b/src/gpgpu-sim/power_stat.h @@ -73,7 +73,7 @@ struct shader_core_power_stats_pod { class power_core_stat_t : public shader_core_power_stats_pod { public: - power_core_stat_t(const struct shader_core_config *shader_config, shader_core_stats *core_stats); + power_core_stat_t(const shader_core_config *shader_config, shader_core_stats *core_stats); void visualizer_print( gzFile visualizer_file ); void print (FILE *fout); void init(); @@ -113,7 +113,7 @@ struct mem_power_stats_pod{ class power_mem_stat_t : public mem_power_stats_pod{ public: - power_mem_stat_t(const struct memory_config *mem_config, const struct shader_core_config *shdr_config, memory_stats_t *mem_stats, shader_core_stats *shdr_stats); + power_mem_stat_t(const struct memory_config *mem_config, const shader_core_config *shdr_config, memory_stats_t *mem_stats, shader_core_stats *shdr_stats); void visualizer_print( gzFile visualizer_file ); void print (FILE *fout) const; void init(); @@ -128,7 +128,7 @@ private: class power_stat_t { public: - power_stat_t( const struct shader_core_config *shader_config,float * average_pipeline_duty_cycle,float * active_sms,shader_core_stats * shader_stats, const struct memory_config *mem_config,memory_stats_t * memory_stats); + power_stat_t( const shader_core_config *shader_config,float * average_pipeline_duty_cycle,float * active_sms,shader_core_stats * shader_stats, const struct memory_config *mem_config,memory_stats_t * memory_stats); void visualizer_print( gzFile visualizer_file ); void print (FILE *fout) const; void save_stats(){ diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index 69b619a..4d12068 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -46,6 +46,7 @@ #include #include "traffic_breakdown.h" #include "shader_trace.h" +#include "../../libcuda/gpgpu_context.h" #define PRIORITIZE_MSHR_OVER_WB 1 #define MAX(a,b) (((a)>(b))?(a):(b)) @@ -69,7 +70,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, class simt_core_cluster *cluster, unsigned shader_id, unsigned tpc_id, - const struct shader_core_config *config, + const shader_core_config *config, const struct memory_config *mem_config, shader_core_stats *stats ) : core_t( gpu, NULL, config->warp_size, config->n_thread_per_shader ), @@ -3018,7 +3019,7 @@ void shader_core_config::set_pipeline_latency() { * [3] MAD * [4] DIV */ - sscanf(opcode_latency_int, "%u,%u,%u,%u,%u", + sscanf(gpgpu_ctx->func_sim->opcode_latency_int, "%u,%u,%u,%u,%u", &int_latency[0],&int_latency[1],&int_latency[2], &int_latency[3],&int_latency[4]); sscanf(opcode_latency_fp, "%u,%u,%u,%u,%u", @@ -3786,7 +3787,7 @@ void opndcoll_rfu_t::collector_unit_t::dispatch() simt_core_cluster::simt_core_cluster( class gpgpu_sim *gpu, unsigned cluster_id, - const struct shader_core_config *config, + const shader_core_config *config, const struct memory_config *mem_config, shader_core_stats *stats, class memory_stats_t *mstats ) diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index 25b9607..e0cefac 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -69,6 +69,8 @@ #define WRITE_MASK_SIZE 8 +class gpgpu_context; + enum exec_unit_type_t { NONE = 0, @@ -294,7 +296,7 @@ typedef std::bitset warp_set_t; int register_bank(int regnum, int wid, unsigned num_banks, unsigned bank_warp_shift, bool sub_core_model, unsigned banks_per_sched, unsigned sched_id ); class shader_core_ctx; -struct shader_core_config; +class shader_core_config; class shader_core_stats; enum scheduler_prioritization_type @@ -1032,7 +1034,7 @@ struct ifetch_buffer_t { unsigned m_warp_id; }; -struct shader_core_config; +class shader_core_config; class simd_function_unit { public: @@ -1362,10 +1364,12 @@ const char* const pipeline_stage_name_decode[] = { "N_PIPELINE_STAGES" }; -struct shader_core_config : public core_config +class shader_core_config : public core_config { - shader_core_config(){ + public: + shader_core_config(gpgpu_context* ctx){ pipeline_widths_string = NULL; + gpgpu_ctx = ctx; } void init() @@ -1425,6 +1429,8 @@ struct shader_core_config : public core_config unsigned cid_to_sid( unsigned cid, unsigned cluster_id ) const { return cluster_id*n_simt_cores_per_cluster + cid; } void set_pipeline_latency(); + // backward pointer + class gpgpu_context* gpgpu_ctx; // data char *gpgpu_shader_core_pipeline_opt; bool gpgpu_perfect_mem; @@ -1770,7 +1776,7 @@ public: class simt_core_cluster *cluster, unsigned shader_id, unsigned tpc_id, - const struct shader_core_config *config, + const shader_core_config *config, const struct memory_config *mem_config, shader_core_stats *stats ); @@ -2065,7 +2071,7 @@ class simt_core_cluster { public: simt_core_cluster( class gpgpu_sim *gpu, unsigned cluster_id, - const struct shader_core_config *config, + const shader_core_config *config, const struct memory_config *mem_config, shader_core_stats *stats, memory_stats_t *mstats ); diff --git a/src/gpgpusim_entrypoint.cc b/src/gpgpusim_entrypoint.cc index b54f20c..5018305 100644 --- a/src/gpgpusim_entrypoint.cc +++ b/src/gpgpusim_entrypoint.cc @@ -218,10 +218,10 @@ gpgpu_sim *gpgpu_context::gpgpu_ptx_sim_init_perf() option_parser_t opp = option_parser_create(); ptx_reg_options(opp); - ptx_opcocde_latency_options(opp); + func_sim->ptx_opcocde_latency_options(opp); icnt_reg_options(opp); - GPGPUsim_ctx_ptr()->g_the_gpu_config = new gpgpu_sim_config(); + GPGPUsim_ctx_ptr()->g_the_gpu_config = new gpgpu_sim_config(this); GPGPUsim_ctx_ptr()->g_the_gpu_config->reg_options(opp); // register GPU microrachitecture options option_parser_cmdline(opp, sg_argc, sg_argv); // parse configuration options -- cgit v1.3 From 87726d32ada00fcd93f2cf24ccae4ba593c4f9ec Mon Sep 17 00:00:00 2001 From: Mengchi Zhang Date: Wed, 10 Jul 2019 15:03:16 -0400 Subject: Move g_pc_to_finfo Signed-off-by: Mengchi Zhang --- src/abstract_hardware_model.cc | 3 ++- src/abstract_hardware_model.h | 13 ++++++++----- src/cuda-sim/cuda-sim.cc | 7 +++---- src/cuda-sim/cuda-sim.h | 5 +++-- src/cuda-sim/ptx_ir.cc | 12 +++++++----- src/cuda-sim/ptx_ir.h | 9 +++++++-- src/cuda-sim/ptx_parser.cc | 2 +- src/gpgpu-sim/shader.cc | 4 ++-- src/gpgpu-sim/shader.h | 2 +- 9 files changed, 34 insertions(+), 23 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc index 248e7a5..ef09051 100644 --- a/src/abstract_hardware_model.cc +++ b/src/abstract_hardware_model.cc @@ -39,6 +39,7 @@ #include #include #include +#include "../libcuda/gpgpu_context.h" unsigned mem_access_t::sm_next_access_uid = 0; unsigned warp_inst_t::sm_next_uid = 0; @@ -945,7 +946,7 @@ void simt_stack::print (FILE *fout) const } else { fprintf(fout," " ); } - ptx_print_insn( stack_entry.m_pc, fout ); + m_gpu->gpgpu_ctx->func_sim->ptx_print_insn( stack_entry.m_pc, fout ); fprintf(fout,"\n"); } diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index da29a11..8ef8376 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -347,9 +347,11 @@ public: mutable bool volta_cache_config_set; }; -struct core_config { - core_config() - { +class core_config { + public: + core_config(gpgpu_context* ctx) + { + gpgpu_ctx = ctx; m_valid = false; num_shmem_bank=16; shmem_limited_broadcast = false; @@ -361,6 +363,8 @@ struct core_config { bool m_valid; unsigned warp_size; + // backward pointer + class gpgpu_context* gpgpu_ctx; // off-chip memory request architecture parameters int gpgpu_coalesce_arch; @@ -934,7 +938,7 @@ public: m_empty=true; m_config=NULL; } - warp_inst_t( const core_config *config ) + warp_inst_t( const core_config *config ) { m_uid=0; assert(config->warp_size<=MAX_WARP_SIZE); @@ -1105,7 +1109,6 @@ public: unsigned get_uid() const { return m_uid; } unsigned get_schd_id() const { return m_scheduler_id; } - protected: unsigned m_uid; diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 3d1da62..fb9bc9e 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -220,7 +220,6 @@ void gpgpu_t::gpgpu_ptx_sim_unbindTexture(const struct textureReference* texref) } unsigned g_assemble_code_next_pc=0; -std::map g_pc_to_finfo; std::vector function_info::s_g_pc_to_insn; #define MAX_INST_SIZE 8 /*bytes*/ @@ -257,7 +256,7 @@ void function_info::ptx_assemble() const symbol *l = pI->get_label(); labels[l->name()] = n; } else { - g_pc_to_finfo[PC] = this; + gpgpu_ctx->func_sim->g_pc_to_finfo[PC] = this; m_instr_mem[n] = pI; s_g_pc_to_insn.push_back(pI); assert(pI == s_g_pc_to_insn[PC]); @@ -497,7 +496,7 @@ void gpgpu_t::gpu_memset( size_t dst_start_addr, int c, size_t count ) } } -void ptx_print_insn( address_type pc, FILE *fp ) +void cuda_sim::ptx_print_insn( address_type pc, FILE *fp ) { std::map::iterator f = g_pc_to_finfo.find(pc); if( f == g_pc_to_finfo.end() ) { @@ -509,7 +508,7 @@ void ptx_print_insn( address_type pc, FILE *fp ) finfo->print_insn(pc,fp); } -std::string ptx_get_insn_str( address_type pc ) +std::string cuda_sim::ptx_get_insn_str( address_type pc ) { std::map::iterator f = g_pc_to_finfo.find(pc); if( f == g_pc_to_finfo.end() ) { diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h index 5c95100..c578524 100644 --- a/src/cuda-sim/cuda-sim.h +++ b/src/cuda-sim/cuda-sim.h @@ -61,8 +61,6 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel, bool functionalSimulationMode = false); const warp_inst_t *ptx_fetch_inst( address_type pc ); const struct gpgpu_ptx_sim_info* ptx_sim_kernel_info(const class function_info *kernel); -void ptx_print_insn( address_type pc, FILE *fp ); -std::string ptx_get_insn_str( address_type pc ); /*! @@ -162,6 +160,7 @@ class cuda_sim { void ** g_inst_op_classification_stat; std::set g_globals; std::set g_constants; + std::map g_pc_to_finfo; // backward pointer class gpgpu_context* gpgpu_ctx; //global functions @@ -181,6 +180,8 @@ class cuda_sim { struct rec_pts find_reconvergence_points( function_info *finfo ); address_type get_converge_point( address_type pc ); void gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to, gpgpu_t *gpu ); + void ptx_print_insn( address_type pc, FILE *fp ); + std::string ptx_get_insn_str( address_type pc ); }; #endif diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index 1bd409e..c537091 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -83,8 +83,9 @@ symbol_table::symbol_table() assert(0); } -symbol_table::symbol_table( const char *scope_name, unsigned entry_point, symbol_table *parent ) +symbol_table::symbol_table( const char *scope_name, unsigned entry_point, symbol_table *parent, gpgpu_context* ctx ) { + gpgpu_ctx = ctx; m_scope_name = std::string(scope_name); m_reg_allocator=0; m_shared_next = 0; @@ -183,7 +184,7 @@ symbol_table* symbol_table::start_inst_group() { //previous added assert(m_inst_group_symtab.find(std::string(inst_group_name)) == m_inst_group_symtab.end()); - symbol_table *sym_table = new symbol_table(inst_group_name, 3/*inst group*/, this ); + symbol_table *sym_table = new symbol_table(inst_group_name, 3/*inst group*/, this, gpgpu_ctx ); sym_table->m_global_next = m_global_next; sym_table->m_shared_next = m_shared_next; @@ -221,7 +222,7 @@ bool symbol_table::add_function_decl( const char *name, int entry_point, functio *func_info = m_function_info_lookup[key]; prior_decl = true; } else { - *func_info = new function_info(entry_point); + *func_info = new function_info(entry_point, gpgpu_ctx); (*func_info)->set_name(name); (*func_info)->set_maxnt_id(0); m_function_info_lookup[key] = *func_info; @@ -232,7 +233,7 @@ bool symbol_table::add_function_decl( const char *name, int entry_point, functio *sym_table = m_function_symtab_lookup[key]; } else { assert( !prior_decl ); - *sym_table = new symbol_table( "", entry_point, this ); + *sym_table = new symbol_table( "", entry_point, this, gpgpu_ctx ); // Initial setup code to support a register represented as "_". // This register is used when an instruction operand is @@ -1373,8 +1374,9 @@ std::string ptx_instruction::to_string() const unsigned function_info::sm_next_uid = 1; -function_info::function_info(int entry_point ) +function_info::function_info(int entry_point, gpgpu_context* ctx ) { + gpgpu_ctx = ctx; m_uid = sm_next_uid++; m_entry_point = (entry_point==1)?true:false; m_extern = (entry_point==2)?true:false; diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 1604551..babd54b 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -306,7 +306,7 @@ private: class symbol_table { public: symbol_table(); - symbol_table( const char *scope_name, unsigned entry_point, symbol_table *parent ); + symbol_table( const char *scope_name, unsigned entry_point, symbol_table *parent, gpgpu_context* ctx); void set_name( const char *name ); const ptx_version &get_ptx_version() const; unsigned get_sm_target() const; @@ -348,6 +348,9 @@ public: symbol_table* start_inst_group(); symbol_table* end_inst_group(); + // backward pointer + class gpgpu_context* gpgpu_ctx; + private: unsigned m_reg_allocator; unsigned m_shared_next; @@ -1233,7 +1236,7 @@ private: class function_info { public: - function_info(int entry_point ); + function_info(int entry_point, gpgpu_context* ctx ); const ptx_version &get_ptx_version() const { return m_symtab->get_ptx_version(); } unsigned get_sm_target() const { return m_symtab->get_sm_target(); } bool is_extern() const { return m_extern; } @@ -1403,6 +1406,8 @@ public: void set_maxnt_id(unsigned maxthreads) { maxnt_id = maxthreads;} unsigned get_maxnt_id() { return maxnt_id;} + // backward pointer + class gpgpu_context* gpgpu_ctx; private: unsigned maxnt_id; diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index 269ec4d..5a94679 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -112,7 +112,7 @@ symbol_table * gpgpu_context::init_parser( const char *ptx_filename ) { g_filename = strdup(ptx_filename); if (g_global_allfiles_symbol_table == NULL) { - g_global_allfiles_symbol_table = new symbol_table("global_allfiles", 0, NULL); + g_global_allfiles_symbol_table = new symbol_table("global_allfiles", 0, NULL, this); ptx_parser->g_global_symbol_table = ptx_parser->g_current_symbol_table = g_global_allfiles_symbol_table; } /*else { diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index 69790fc..6cd6d8f 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -1025,7 +1025,7 @@ void scheduler_unit::cycle() m_simt_stack[warp_id]->get_pdom_stack_top_info(&pc,&rpc); SCHED_DPRINTF( "Warp (warp_id %u, dynamic_warp_id %u) has valid instruction (%s)\n", (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id(), - ptx_get_insn_str( pc).c_str() ); + m_shader->m_config->gpgpu_ctx->func_sim->ptx_get_insn_str( pc).c_str() ); if( pI ) { assert(valid); if( pc != pI->pc ) { @@ -2690,7 +2690,7 @@ void warp_inst_t::print( FILE *fout ) const for (unsigned j=0; jwarp_size; j++) fprintf(fout, "%c", (active(j)?'1':'0') ); fprintf(fout, "]: "); - ptx_print_insn( pc, fout ); + m_config->gpgpu_ctx->func_sim->ptx_print_insn( pc, fout ); fprintf(fout, "\n"); } void shader_core_ctx::incexecstat(warp_inst_t *&inst) diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index e0cefac..2837f1b 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -1367,7 +1367,7 @@ const char* const pipeline_stage_name_decode[] = { class shader_core_config : public core_config { public: - shader_core_config(gpgpu_context* ctx){ + shader_core_config(gpgpu_context* ctx):core_config(ctx){ pipeline_widths_string = NULL; gpgpu_ctx = ctx; } -- cgit v1.3 From 67950eb12eec495b1976934acc763db481d955f8 Mon Sep 17 00:00:00 2001 From: Mengchi Zhang Date: Mon, 15 Jul 2019 11:58:10 -0400 Subject: Move operand_info::sm_next_uid Signed-off-by: Mengchi Zhang --- libcuda/gpgpu_context.h | 2 ++ src/cuda-sim/ptx_ir.cc | 20 ++++++++++------- src/cuda-sim/ptx_ir.h | 54 ++++++++++++++++++++++++---------------------- src/cuda-sim/ptx_parser.cc | 30 +++++++++++++------------- src/cuda-sim/ptx_parser.h | 2 +- src/cuda-sim/ptx_sim.cc | 8 +++---- 6 files changed, 62 insertions(+), 54 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/libcuda/gpgpu_context.h b/libcuda/gpgpu_context.h index 0dd6311..9c5ae7b 100644 --- a/libcuda/gpgpu_context.h +++ b/libcuda/gpgpu_context.h @@ -13,6 +13,7 @@ class gpgpu_context { g_global_allfiles_symbol_table = NULL; sm_next_access_uid=0; warp_inst_sm_next_uid=0; + operand_info_sm_next_uid = 1; api = new cuda_runtime_api(this); ptxinfo = new ptxinfo_data(this); ptx_parser = new ptx_recognizer(this); @@ -25,6 +26,7 @@ class gpgpu_context { const char *g_filename; unsigned sm_next_access_uid; unsigned warp_inst_sm_next_uid; + unsigned operand_info_sm_next_uid;//uid for operand_info // objects pointers for each file cuda_runtime_api* api; ptxinfo_data* ptxinfo; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index c537091..0fa29eb 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -39,6 +39,7 @@ typedef void * yyscan_t; #include "assert.h" #include "cuda-sim.h" +#include "../../libcuda/gpgpu_context.h" #define STR_SIZE 1024 @@ -322,11 +323,9 @@ void symbol_table::dump() printf("\n"); } -unsigned operand_info::sm_next_uid=1; - unsigned operand_info::get_uid() { - unsigned result = sm_next_uid++; + unsigned result = (gpgpu_ctx->operand_info_sm_next_uid)++; return result; } @@ -1015,7 +1014,7 @@ void copy_buffer_to_frame(ptx_thread_info * thread, const arg_buffer_t &a) { if( a.is_reg() ) { ptx_reg_t value = a.get_reg(); - operand_info dst_reg = operand_info(a.get_dst()); + operand_info dst_reg = operand_info(a.get_dst(), thread->get_gpu()->gpgpu_ctx); thread->set_reg(dst_reg.get_symbol(),value); } else { const void *buffer = a.get_param_buffer(); @@ -1039,7 +1038,8 @@ void copy_buffer_list_into_frame(ptx_thread_info * thread, arg_buffer_list_t &ar static std::list check_operands( int opcode, const std::list &scalar_type, - const std::list &operands ) + const std::list &operands, + gpgpu_context* ctx) { static int g_warn_literal_operands_two_type_inst; if( (opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || (opcode == TEX_OP) || (opcode==MMA_OP) || (opcode == DP4A_OP)) { @@ -1066,7 +1066,7 @@ static std::list check_operands( int opcode, if( (op.get_type() == double_op_t) && (inst_type == F32_TYPE) ) { ptx_reg_t v = op.get_literal_value(); float u = (float)v.f64; - operand_info n(u); + operand_info n(u, ctx); result.push_back(n); } else { result.push_back(op); @@ -1097,7 +1097,7 @@ ptx_instruction::ptx_instruction( int opcode, unsigned line, const char *source, const core_config *config, - gpgpu_context* ctx ) : warp_inst_t(config) + gpgpu_context* ctx ) : warp_inst_t(config), m_return_var(ctx) { gpgpu_ctx = ctx; m_uid = ++g_num_ptx_inst_uid; @@ -1107,7 +1107,7 @@ ptx_instruction::ptx_instruction( int opcode, m_neg_pred = neg_pred; m_pred_mod = pred_mod; m_label = label; - const std::list checked_operands = check_operands(opcode,scalar_type,operands); + const std::list checked_operands = check_operands(opcode,scalar_type,operands, ctx); m_operands.insert(m_operands.begin(), checked_operands.begin(), checked_operands.end() ); m_return_var = return_var; m_options = options; @@ -1371,6 +1371,10 @@ std::string ptx_instruction::to_string() const m_source.c_str() ); return std::string( buf ); } +operand_info ptx_instruction::get_pred() const +{ + return operand_info( m_pred, gpgpu_ctx); +} unsigned function_info::sm_next_uid = 1; diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index babd54b..8ed94d9 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -377,9 +377,9 @@ private: class operand_info { public: - operand_info() + operand_info(gpgpu_context* ctx) { - init(); + init(ctx); m_is_non_arch_reg = false; m_addr_space = undefined_space; m_operand_lohi = 0; @@ -392,9 +392,9 @@ public: m_addr_offset = 0; m_value.m_symbolic=NULL; } - operand_info( const symbol *addr ) + operand_info( const symbol *addr, gpgpu_context* ctx ) { - init(); + init(ctx); m_is_non_arch_reg = false; m_addr_space = undefined_space; m_operand_lohi = 0; @@ -435,9 +435,9 @@ public: m_is_return_var = false; m_immediate_address=false; } - operand_info( const symbol *addr1, const symbol *addr2 ) + operand_info( const symbol *addr1, const symbol *addr2, gpgpu_context* ctx ) { - init(); + init(ctx); m_is_non_arch_reg = false; m_addr_space = undefined_space; m_operand_lohi = 0; @@ -462,9 +462,9 @@ public: m_is_return_var = false; m_immediate_address=false; } - operand_info( int builtin_id, int dim_mod ) + operand_info( int builtin_id, int dim_mod, gpgpu_context* ctx ) { - init(); + init(ctx); m_is_non_arch_reg = false; m_addr_space = undefined_space; m_operand_lohi = 0; @@ -481,9 +481,9 @@ public: m_is_return_var = false; m_immediate_address=false; } - operand_info( const symbol *addr, int offset ) + operand_info( const symbol *addr, int offset, gpgpu_context* ctx ) { - init(); + init(ctx); m_is_non_arch_reg = false; m_addr_space = undefined_space; m_operand_lohi = 0; @@ -500,9 +500,9 @@ public: m_is_return_var = false; m_immediate_address=false; } - operand_info( unsigned x ) + operand_info( unsigned x, gpgpu_context* ctx ) { - init(); + init(ctx); m_is_non_arch_reg = false; m_addr_space = undefined_space; m_operand_lohi = 0; @@ -519,9 +519,9 @@ public: m_is_return_var = false; m_immediate_address=true; } - operand_info( int x ) + operand_info( int x, gpgpu_context* ctx ) { - init(); + init(ctx); m_is_non_arch_reg = false; m_addr_space = undefined_space; m_operand_lohi = 0; @@ -538,9 +538,9 @@ public: m_is_return_var = false; m_immediate_address=false; } - operand_info( float x ) + operand_info( float x, gpgpu_context* ctx ) { - init(); + init(ctx); m_is_non_arch_reg = false; m_addr_space = undefined_space; m_operand_lohi = 0; @@ -557,9 +557,9 @@ public: m_is_return_var = false; m_immediate_address=false; } - operand_info( double x ) + operand_info( double x, gpgpu_context* ctx ) { - init(); + init(ctx); m_is_non_arch_reg = false; m_addr_space = undefined_space; m_operand_lohi = 0; @@ -576,9 +576,9 @@ public: m_is_return_var = false; m_immediate_address=false; } - operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4 ) + operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4, gpgpu_context* ctx ) { - init(); + init(ctx); m_is_non_arch_reg = false; m_addr_space = undefined_space; m_operand_lohi = 0; @@ -603,9 +603,9 @@ public: m_is_return_var = false; m_immediate_address=false; } - operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4 ,const symbol *s5,const symbol *s6,const symbol *s7, const symbol *s8) + operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4 ,const symbol *s5,const symbol *s6,const symbol *s7, const symbol *s8, gpgpu_context* ctx) { - init(); + init(ctx); m_is_non_arch_reg = false; m_addr_space = undefined_space; m_operand_lohi = 0; @@ -631,8 +631,9 @@ public: m_immediate_address=false; } - void init() + void init(gpgpu_context* ctx) { + gpgpu_ctx = ctx; m_uid=(unsigned)-1; m_valid=false; m_vector=false; @@ -842,6 +843,7 @@ public: bool is_non_arch_reg() const { return m_is_non_arch_reg; } private: + gpgpu_context* gpgpu_ctx; unsigned m_uid; bool m_valid; bool m_vector; @@ -957,7 +959,7 @@ public: unsigned source_line() const { return m_source_line;} unsigned get_num_operands() const { return m_operands.size();} bool has_pred() const { return m_pred != NULL;} - operand_info get_pred() const { return operand_info( m_pred );} + operand_info get_pred() const; bool get_pred_neg() const { return m_neg_pred;} int get_pred_mod() const { return m_pred_mod;} const char *get_source() const { return m_source.c_str();} @@ -1448,14 +1450,14 @@ private: class arg_buffer_t { public: - arg_buffer_t() + arg_buffer_t(gpgpu_context* ctx) : m_src_op(ctx) { m_is_reg=false; m_is_param=false; m_param_value=NULL; m_reg_value=ptx_reg_t(); } - arg_buffer_t( const arg_buffer_t &another ) + arg_buffer_t( const arg_buffer_t &another, gpgpu_context* ctx ) : m_src_op(ctx) { make_copy(another); } diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index 1d38edf..81b70af 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -102,7 +102,7 @@ void ptx_recognizer::init_instruction_state() g_opcode = -1; g_options.clear(); g_wmma_options.clear(); - g_return_var = operand_info(); + g_return_var = operand_info(gpgpu_ctx); init_directive_state(); } @@ -689,7 +689,7 @@ void ptx_recognizer::add_double_operand( const char *d1, const char *d2 ) const symbol *s1 = g_current_symbol_table->lookup(d1); const symbol *s2 = g_current_symbol_table->lookup(d2); parse_assert( s1 != NULL && s2 != NULL, "component(s) missing declarations."); - g_operands.push_back( operand_info(s1,s2) ); + g_operands.push_back( operand_info(s1,s2,gpgpu_ctx) ); } void ptx_recognizer::add_1vector_operand( const char *d1 ) @@ -698,7 +698,7 @@ void ptx_recognizer::add_1vector_operand( const char *d1 ) PTX_PARSE_DPRINTF("add_1vector_operand"); const symbol *s1 = g_current_symbol_table->lookup(d1); parse_assert( s1 != NULL, "component(s) missing declarations."); - g_operands.push_back( operand_info(s1,NULL,NULL,NULL) ); + g_operands.push_back( operand_info(s1,NULL,NULL,NULL,gpgpu_ctx) ); } void ptx_recognizer::add_2vector_operand( const char *d1, const char *d2 ) @@ -707,7 +707,7 @@ void ptx_recognizer::add_2vector_operand( const char *d1, const char *d2 ) const symbol *s1 = g_current_symbol_table->lookup(d1); const symbol *s2 = g_current_symbol_table->lookup(d2); parse_assert( s1 != NULL && s2 != NULL, "v2 component(s) missing declarations."); - g_operands.push_back( operand_info(s1,s2,NULL,NULL) ); + g_operands.push_back( operand_info(s1,s2,NULL,NULL,gpgpu_ctx) ); } void ptx_recognizer::add_3vector_operand( const char *d1, const char *d2, const char *d3 ) @@ -717,7 +717,7 @@ void ptx_recognizer::add_3vector_operand( const char *d1, const char *d2, const const symbol *s2 = g_current_symbol_table->lookup(d2); const symbol *s3 = g_current_symbol_table->lookup(d3); parse_assert( s1 != NULL && s2 != NULL && s3 != NULL, "v3 component(s) missing declarations."); - g_operands.push_back( operand_info(s1,s2,s3,NULL) ); + g_operands.push_back( operand_info(s1,s2,s3,NULL,gpgpu_ctx) ); } void ptx_recognizer::add_4vector_operand( const char *d1, const char *d2, const char *d3, const char *d4 ) @@ -732,7 +732,7 @@ void ptx_recognizer::add_4vector_operand( const char *d1, const char *d2, const if ( s2 == null_op ) s2 = NULL; if ( s3 == null_op ) s3 = NULL; if ( s4 == null_op ) s4 = NULL; - g_operands.push_back( operand_info(s1,s2,s3,s4) ); + g_operands.push_back( operand_info(s1,s2,s3,s4,gpgpu_ctx) ); } void ptx_recognizer::add_8vector_operand( const char *d1, const char *d2, const char *d3, const char *d4,const char *d5,const char *d6,const char *d7,const char *d8 ) { @@ -754,13 +754,13 @@ void ptx_recognizer::add_8vector_operand( const char *d1, const char *d2, const if ( s6 == null_op ) s6 = NULL; if ( s7 == null_op ) s7 = NULL; if ( s8 == null_op ) s8 = NULL; - g_operands.push_back( operand_info(s1,s2,s3,s4,s5,s6,s7,s8) ); + g_operands.push_back( operand_info(s1,s2,s3,s4,s5,s6,s7,s8,gpgpu_ctx) ); } void ptx_recognizer::add_builtin_operand( int builtin, int dim_modifier ) { PTX_PARSE_DPRINTF("add_builtin_operand"); - g_operands.push_back( operand_info(builtin,dim_modifier) ); + g_operands.push_back( operand_info(builtin,dim_modifier,gpgpu_ctx) ); } void ptx_recognizer::add_memory_operand() @@ -883,19 +883,19 @@ void ptx_recognizer::change_operand_neg( ) void ptx_recognizer::add_literal_int( int value ) { PTX_PARSE_DPRINTF("add_literal_int"); - g_operands.push_back( operand_info(value) ); + g_operands.push_back( operand_info(value,gpgpu_ctx) ); } void ptx_recognizer::add_literal_float( float value ) { PTX_PARSE_DPRINTF("add_literal_float"); - g_operands.push_back( operand_info(value) ); + g_operands.push_back( operand_info(value,gpgpu_ctx) ); } void ptx_recognizer::add_literal_double( double value ) { PTX_PARSE_DPRINTF("add_literal_double"); - g_operands.push_back( operand_info(value) ); + g_operands.push_back( operand_info(value,gpgpu_ctx) ); } void ptx_recognizer::add_scalar_operand( const char *identifier ) @@ -911,7 +911,7 @@ void ptx_recognizer::add_scalar_operand( const char *identifier ) parse_error( msg.c_str() ); } } - g_operands.push_back( operand_info(s) ); + g_operands.push_back( operand_info(s,gpgpu_ctx) ); } void ptx_recognizer::add_neg_pred_operand( const char *identifier ) @@ -921,7 +921,7 @@ void ptx_recognizer::add_neg_pred_operand( const char *identifier ) if ( s == NULL ) { s = g_current_symbol_table->add_variable(identifier,NULL,1,gpgpu_ctx->g_filename,ptx_get_lineno(scanner)); } - operand_info op(s); + operand_info op(s, gpgpu_ctx); op.set_neg_pred(); g_operands.push_back( op ); } @@ -934,13 +934,13 @@ void ptx_recognizer::add_address_operand( const char *identifier, int offset ) std::string msg = std::string("operand \"") + identifier + "\" has no declaration."; parse_error( msg.c_str() ); } - g_operands.push_back( operand_info(s,offset) ); + g_operands.push_back( operand_info(s,offset,gpgpu_ctx) ); } void ptx_recognizer::add_address_operand2( int offset ) { PTX_PARSE_DPRINTF("add_address_operand"); - g_operands.push_back( operand_info((unsigned)offset) ); + g_operands.push_back( operand_info((unsigned)offset,gpgpu_ctx) ); } void ptx_recognizer::add_array_initializer() diff --git a/src/cuda-sim/ptx_parser.h b/src/cuda-sim/ptx_parser.h index 1e559a2..11a3d20 100644 --- a/src/cuda-sim/ptx_parser.h +++ b/src/cuda-sim/ptx_parser.h @@ -35,7 +35,7 @@ class gpgpu_context; typedef void * yyscan_t; class ptx_recognizer { public: - ptx_recognizer( gpgpu_context* ctx ) { + ptx_recognizer( gpgpu_context* ctx ) : g_return_var(ctx) { scanner = NULL; g_size = -1; g_add_identifier_cached__identifier = NULL; diff --git a/src/cuda-sim/ptx_sim.cc b/src/cuda-sim/ptx_sim.cc index b6cf4bd..62bb529 100644 --- a/src/cuda-sim/ptx_sim.cc +++ b/src/cuda-sim/ptx_sim.cc @@ -422,9 +422,9 @@ bool ptx_thread_info::callstack_pop() assert( !((rv_src != NULL) ^ (rv_dst != NULL)) ); // ensure caller and callee agree on whether there is a return value // read return value from callee frame - arg_buffer_t buffer; + arg_buffer_t buffer(m_gpu->gpgpu_ctx); if( rv_src != NULL ) - buffer = copy_arg_to_buffer(this, operand_info(rv_src), rv_dst ); + buffer = copy_arg_to_buffer(this, operand_info(rv_src, m_gpu->gpgpu_ctx), rv_dst ); m_symbol_table = m_callstack.back().m_symbol_table; m_NPC = m_callstack.back().m_PC; @@ -456,9 +456,9 @@ bool ptx_thread_info::callstack_pop_plus() assert( !((rv_src != NULL) ^ (rv_dst != NULL)) ); // ensure caller and callee agree on whether there is a return value // read return value from callee frame - arg_buffer_t buffer; + arg_buffer_t buffer(m_gpu->gpgpu_ctx); if( rv_src != NULL ) - buffer = copy_arg_to_buffer(this, operand_info(rv_src), rv_dst ); + buffer = copy_arg_to_buffer(this, operand_info(rv_src, m_gpu->gpgpu_ctx), rv_dst ); m_symbol_table = m_callstack.back().m_symbol_table; m_NPC = m_callstack.back().m_PC; -- cgit v1.3 From 3b88f9db669d8ec9ffc477352dcfd2a3d423781f Mon Sep 17 00:00:00 2001 From: Mengchi Zhang Date: Mon, 15 Jul 2019 13:38:38 -0400 Subject: Move g_num_ptx_inst_uid Signed-off-by: Mengchi Zhang --- libcuda/gpgpu_context.h | 2 ++ src/cuda-sim/instructions.cc | 1 - src/cuda-sim/ptx_ir.cc | 2 +- src/cuda-sim/ptx_ir.h | 2 -- 4 files changed, 3 insertions(+), 4 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/libcuda/gpgpu_context.h b/libcuda/gpgpu_context.h index 337ebb2..909f267 100644 --- a/libcuda/gpgpu_context.h +++ b/libcuda/gpgpu_context.h @@ -16,6 +16,7 @@ class gpgpu_context { warp_inst_sm_next_uid=0; operand_info_sm_next_uid = 1; kernel_info_m_next_uid = 1; + g_num_ptx_inst_uid = 0; api = new cuda_runtime_api(this); ptxinfo = new ptxinfo_data(this); ptx_parser = new ptx_recognizer(this); @@ -31,6 +32,7 @@ class gpgpu_context { unsigned warp_inst_sm_next_uid; unsigned operand_info_sm_next_uid;//uid for operand_info unsigned kernel_info_m_next_uid;//uid for kernel_info_t + unsigned g_num_ptx_inst_uid; //uid for ptx inst inside ptx_instruction // objects pointers for each file cuda_runtime_api* api; ptxinfo_data* ptxinfo; diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 565340c..4d4a80d 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -60,7 +60,6 @@ class ptx_recognizer; using half_float::half; -unsigned ptx_instruction::g_num_ptx_inst_uid=0; bool debug_tensorcore = 0; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index 0fa29eb..849cc5d 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -1100,7 +1100,7 @@ ptx_instruction::ptx_instruction( int opcode, gpgpu_context* ctx ) : warp_inst_t(config), m_return_var(ctx) { gpgpu_ctx = ctx; - m_uid = ++g_num_ptx_inst_uid; + m_uid = ++(ctx->g_num_ptx_inst_uid); m_PC = 0; m_opcode = opcode; m_pred = pred; diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 8ed94d9..1ffbebc 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -878,7 +878,6 @@ private: }; extern const char *g_opcode_string[]; -extern unsigned g_num_ptx_inst_uid; struct basic_block_t { basic_block_t( unsigned ID, ptx_instruction *begin, ptx_instruction *end, bool entry, bool ex) { @@ -1194,7 +1193,6 @@ private: virtual void pre_decode(); friend class function_info; - static unsigned g_num_ptx_inst_uid; // backward pointer class gpgpu_context* gpgpu_ctx; }; -- cgit v1.3 From 971722f51189e80034a9c80a4846a4ec045f0d4d Mon Sep 17 00:00:00 2001 From: Mengchi Zhang Date: Mon, 15 Jul 2019 13:57:48 -0400 Subject: Move g_ptx_cta_info_uid and symbol::sm_next_uid Signed-off-by: Mengchi Zhang --- libcuda/gpgpu_context.h | 4 ++++ src/cuda-sim/ptx_ir.cc | 8 +++----- src/cuda-sim/ptx_ir.h | 6 +++--- src/cuda-sim/ptx_sim.cc | 3 +-- 4 files changed, 11 insertions(+), 10 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/libcuda/gpgpu_context.h b/libcuda/gpgpu_context.h index 909f267..e168850 100644 --- a/libcuda/gpgpu_context.h +++ b/libcuda/gpgpu_context.h @@ -17,6 +17,8 @@ class gpgpu_context { operand_info_sm_next_uid = 1; kernel_info_m_next_uid = 1; g_num_ptx_inst_uid = 0; + g_ptx_cta_info_uid = 1; + symbol_sm_next_uid = 1; api = new cuda_runtime_api(this); ptxinfo = new ptxinfo_data(this); ptx_parser = new ptx_recognizer(this); @@ -33,6 +35,8 @@ class gpgpu_context { unsigned operand_info_sm_next_uid;//uid for operand_info unsigned kernel_info_m_next_uid;//uid for kernel_info_t unsigned g_num_ptx_inst_uid; //uid for ptx inst inside ptx_instruction + unsigned long long g_ptx_cta_info_uid; + unsigned symbol_sm_next_uid; //uid for symbol // objects pointers for each file cuda_runtime_api* api; ptxinfo_data* ptxinfo; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index 849cc5d..3719fb2 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -43,11 +43,9 @@ typedef void * yyscan_t; #define STR_SIZE 1024 -unsigned symbol::sm_next_uid = 1; - unsigned symbol::get_uid() { - unsigned result = sm_next_uid++; + unsigned result = (gpgpu_ctx->symbol_sm_next_uid)++; return result; } @@ -152,7 +150,7 @@ symbol *symbol_table::add_variable( const char *identifier, const type_info *typ std::string key(identifier); assert( m_symbols.find(key) == m_symbols.end() ); snprintf(buf,1024,"%s:%u",filename,line); - symbol *s = new symbol(identifier,type,buf,size); + symbol *s = new symbol(identifier,type,buf,size,gpgpu_ctx); m_symbols[ key ] = s; if ( type != NULL && type->get_key().is_global() ) { @@ -173,7 +171,7 @@ void symbol_table::add_function( function_info *func, const char *filename, unsi char buf[1024]; snprintf(buf,1024,"%s:%u",filename,linenumber); type_info *type = add_type( func ); - symbol *s = new symbol(func->get_name().c_str(),type,buf,0); + symbol *s = new symbol(func->get_name().c_str(),type,buf,0,gpgpu_ctx); s->set_function(func); m_symbols[ func->get_name() ] = s; } diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 1ffbebc..2fbda1c 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -152,8 +152,9 @@ class operand_info; class symbol { public: - symbol( const char *name, const type_info *type, const char *location, unsigned size ) + symbol( const char *name, const type_info *type, const char *location, unsigned size, gpgpu_context* ctx ) { + gpgpu_ctx = ctx; m_uid = get_uid(); m_name = name; m_decl_location = location; @@ -273,6 +274,7 @@ public: unsigned uid() const { return m_uid; } private: + gpgpu_context* gpgpu_ctx; unsigned get_uid(); unsigned m_uid; const type_info *m_type; @@ -299,8 +301,6 @@ private: bool m_reg_num_valid; std::list m_initializer; - static unsigned sm_next_uid; - }; class symbol_table { diff --git a/src/cuda-sim/ptx_sim.cc b/src/cuda-sim/ptx_sim.cc index 62bb529..949ee66 100644 --- a/src/cuda-sim/ptx_sim.cc +++ b/src/cuda-sim/ptx_sim.cc @@ -37,7 +37,6 @@ typedef void * yyscan_t; void feature_not_implemented( const char *f ); -unsigned long long g_ptx_cta_info_uid = 1; ptx_cta_info::ptx_cta_info( unsigned sm_idx, gpgpu_context* ctx ) { @@ -45,7 +44,7 @@ ptx_cta_info::ptx_cta_info( unsigned sm_idx, gpgpu_context* ctx ) ctx->func_sim->g_ptx_cta_info_sm_idx_used.insert(sm_idx); m_sm_idx = sm_idx; - m_uid = g_ptx_cta_info_uid++; + m_uid = (ctx->g_ptx_cta_info_uid)++; m_bar_threads = 0; gpgpu_ctx = ctx; } -- cgit v1.3 From 6ec4563b11fe99e539eb83412acaadc6f67b05ba Mon Sep 17 00:00:00 2001 From: Mengchi Zhang Date: Mon, 15 Jul 2019 14:04:43 -0400 Subject: Move function_info::sm_next_id Signed-off-by: Mengchi Zhang --- libcuda/gpgpu_context.h | 2 ++ src/cuda-sim/ptx_ir.cc | 3 +-- src/cuda-sim/ptx_ir.h | 2 -- 3 files changed, 3 insertions(+), 4 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/libcuda/gpgpu_context.h b/libcuda/gpgpu_context.h index e168850..ed4f746 100644 --- a/libcuda/gpgpu_context.h +++ b/libcuda/gpgpu_context.h @@ -19,6 +19,7 @@ class gpgpu_context { g_num_ptx_inst_uid = 0; g_ptx_cta_info_uid = 1; symbol_sm_next_uid = 1; + function_info_sm_next_uid = 1; api = new cuda_runtime_api(this); ptxinfo = new ptxinfo_data(this); ptx_parser = new ptx_recognizer(this); @@ -37,6 +38,7 @@ class gpgpu_context { unsigned g_num_ptx_inst_uid; //uid for ptx inst inside ptx_instruction unsigned long long g_ptx_cta_info_uid; unsigned symbol_sm_next_uid; //uid for symbol + unsigned function_info_sm_next_uid; // objects pointers for each file cuda_runtime_api* api; ptxinfo_data* ptxinfo; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index 3719fb2..5fa9379 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -1374,12 +1374,11 @@ operand_info ptx_instruction::get_pred() const return operand_info( m_pred, gpgpu_ctx); } -unsigned function_info::sm_next_uid = 1; function_info::function_info(int entry_point, gpgpu_context* ctx ) { gpgpu_ctx = ctx; - m_uid = sm_next_uid++; + m_uid = (gpgpu_ctx->function_info_sm_next_uid)++; m_entry_point = (entry_point==1)?true:false; m_extern = (entry_point==2)?true:false; num_reconvergence_pairs = 0; diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 2fbda1c..8fc0a06 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -873,7 +873,6 @@ private: bool m_is_return_var; bool m_is_non_arch_reg; - static unsigned sm_next_uid; unsigned get_uid(); }; @@ -1438,7 +1437,6 @@ private: symbol_table *m_symtab; static std::vector s_g_pc_to_insn; // a direct mapping from PC to instruction - static unsigned sm_next_uid; //parameter size for device kernels int m_args_aligned_size; -- cgit v1.3 From 352d2a3336b1c8e5258ca9d92d214973e98837c0 Mon Sep 17 00:00:00 2001 From: Mengchi Zhang Date: Mon, 15 Jul 2019 16:17:07 -0400 Subject: Move s_g_pc_to_insn Signed-off-by: Mengchi Zhang --- libcuda/gpgpu_context.h | 5 +++++ src/abstract_hardware_model.cc | 10 +++++----- src/cuda-sim/cuda-sim.cc | 20 +++++++++----------- src/cuda-sim/cuda-sim.h | 2 -- src/cuda-sim/ptx-stats.cc | 28 ++++++++++++++-------------- src/cuda-sim/ptx-stats.h | 15 ++++++++------- src/cuda-sim/ptx_ir.cc | 8 ++++++++ src/cuda-sim/ptx_ir.h | 9 --------- src/gpgpu-sim/gpu-sim.cc | 2 +- src/gpgpu-sim/mem_latency_stat.cc | 3 ++- src/gpgpu-sim/shader.cc | 4 ++-- src/gpgpu-sim/stat-tool.cc | 33 +++++++++++++++++---------------- src/gpgpu-sim/stat-tool.h | 12 +++++++----- 13 files changed, 78 insertions(+), 73 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/libcuda/gpgpu_context.h b/libcuda/gpgpu_context.h index ed4f746..346a8a4 100644 --- a/libcuda/gpgpu_context.h +++ b/libcuda/gpgpu_context.h @@ -39,6 +39,8 @@ class gpgpu_context { unsigned long long g_ptx_cta_info_uid; unsigned symbol_sm_next_uid; //uid for symbol unsigned function_info_sm_next_uid; + std::vector s_g_pc_to_insn; // a direct mapping from PC to instruction + // objects pointers for each file cuda_runtime_api* api; ptxinfo_data* ptxinfo; @@ -58,6 +60,9 @@ class gpgpu_context { class gpgpu_sim *gpgpu_ptx_sim_init_perf(); struct _cuda_device_id *GPGPUSim_Init(); void ptx_reg_options(option_parser_t opp); + const ptx_instruction* pc_to_instruction(unsigned pc); + const warp_inst_t *ptx_fetch_inst( address_type pc ); + unsigned translate_pc_to_ptxlineno(unsigned pc); }; gpgpu_context* GPGPU_Context(); diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc index 733d602..d8d5fbd 100644 --- a/src/abstract_hardware_model.cc +++ b/src/abstract_hardware_model.cc @@ -426,7 +426,7 @@ void warp_inst_t::generate_mem_accesses() } assert( total_accesses > 0 && total_accesses <= m_config->warp_size ); cycles = total_accesses; // shared memory conflicts modeled as larger initiation interval - ptx_file_line_stats_add_smem_bank_conflict( pc, total_accesses ); + m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_smem_bank_conflict( pc, total_accesses ); break; } @@ -471,7 +471,7 @@ void warp_inst_t::generate_mem_accesses() } if ( space.get_type() == global_space ) { - ptx_file_line_stats_add_uncoalesced_gmem( pc, m_accessq.size() - starting_queue_size ); + m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_uncoalesced_gmem( pc, m_accessq.size() - starting_queue_size ); } m_mem_accesses_created=true; } @@ -706,7 +706,7 @@ void warp_inst_t::completed( unsigned long long cycle ) const { unsigned long long latency = cycle - issue_cycle; assert(latency <= cycle); // underflow detection - ptx_file_line_stats_add_latency(pc, latency * active_count()); + m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_latency(pc, latency * active_count()); } @@ -1110,7 +1110,7 @@ void simt_stack::update( simt_mask_t &thread_done, addr_vector_t &next_pc, addre if (warp_diverged) { - ptx_file_line_stats_add_warp_divergence(top_pc, 1); + m_gpu->gpgpu_ctx->stats->ptx_file_line_stats_add_warp_divergence(top_pc, 1); } } @@ -1157,7 +1157,7 @@ warp_inst_t core_t::getExecuteWarp(unsigned warpId) { unsigned pc,rpc; m_simt_stack[warpId]->get_pdom_stack_top_info(&pc,&rpc); - warp_inst_t wi= *ptx_fetch_inst(pc); + warp_inst_t wi= *(m_gpu->gpgpu_ctx->ptx_fetch_inst(pc)); wi.set_active(m_simt_stack[warpId]->get_active_mask()); return wi; } diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index c06f093..b9e6552 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -215,8 +215,6 @@ void gpgpu_t::gpgpu_ptx_sim_unbindTexture(const struct textureReference* texref) m_NameToTextureInfo.erase(texname); } -std::vector function_info::s_g_pc_to_insn; - #define MAX_INST_SIZE 8 /*bytes*/ void function_info::ptx_assemble() @@ -237,14 +235,14 @@ void function_info::ptx_assemble() addr_t PC = gpgpu_ctx->func_sim->g_assemble_code_next_pc; // globally unique address (across functions) // start function on an aligned address for( unsigned i=0; i < (PC%MAX_INST_SIZE); i++ ) - s_g_pc_to_insn.push_back((ptx_instruction*)NULL); + gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction*)NULL); PC += PC%MAX_INST_SIZE; m_start_PC = PC; addr_t n=0; // offset in m_instr_mem //Why s_g_pc_to_insn.size() is needed to reserve additional memory for insts? reserve is cumulative. //s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + MAX_INST_SIZE*m_instructions.size()); - s_g_pc_to_insn.reserve(MAX_INST_SIZE*m_instructions.size()); + gpgpu_ctx->s_g_pc_to_insn.reserve(MAX_INST_SIZE*m_instructions.size()); for ( i=m_instructions.begin(); i != m_instructions.end(); i++ ) { ptx_instruction *pI = *i; if ( pI->is_label() ) { @@ -253,13 +251,13 @@ void function_info::ptx_assemble() } else { gpgpu_ctx->func_sim->g_pc_to_finfo[PC] = this; m_instr_mem[n] = pI; - s_g_pc_to_insn.push_back(pI); - assert(pI == s_g_pc_to_insn[PC]); + gpgpu_ctx->s_g_pc_to_insn.push_back(pI); + assert(pI == gpgpu_ctx->s_g_pc_to_insn[PC]); pI->set_m_instr_mem_index(n); pI->set_PC(PC); assert( pI->inst_size() <= MAX_INST_SIZE ); for( unsigned i=1; i < pI->inst_size(); i++ ) { - s_g_pc_to_insn.push_back((ptx_instruction*)NULL); + gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction*)NULL); m_instr_mem[n+i]=NULL; } n += pI->inst_size(); @@ -1738,9 +1736,9 @@ const struct gpgpu_ptx_sim_info* ptx_sim_kernel_info(const function_info *kernel return kernel->get_kernel_info(); } -const warp_inst_t *ptx_fetch_inst( address_type pc ) +const warp_inst_t *gpgpu_context::ptx_fetch_inst( address_type pc ) { - return function_info::pc_to_instruction(pc); + return pc_to_instruction(pc); } unsigned ptx_sim_init_thread( kernel_info_t &kernel, @@ -2366,11 +2364,11 @@ void functionalCoreSim::executeWarp(unsigned i, bool &allAtBarrier, bool & someO if(!m_warpAtBarrier[i]&& m_liveThreadCount[i]>0) allAtBarrier = false; } -unsigned translate_pc_to_ptxlineno(unsigned pc) +unsigned gpgpu_context::translate_pc_to_ptxlineno(unsigned pc) { // this function assumes that the kernel fits inside a single PTX file // function_info *pFunc = g_func_info; // assume that the current kernel is the one in query - const ptx_instruction *pInsn = function_info::pc_to_instruction(pc); + const ptx_instruction *pInsn = pc_to_instruction(pc); unsigned ptx_line_number = pInsn->source_line(); return ptx_line_number; diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h index 5bd4cb2..1be3d19 100644 --- a/src/cuda-sim/cuda-sim.h +++ b/src/cuda-sim/cuda-sim.h @@ -58,10 +58,8 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel, unsigned hw_warp_id, gpgpu_t *gpu, bool functionalSimulationMode = false); -const warp_inst_t *ptx_fetch_inst( address_type pc ); const struct gpgpu_ptx_sim_info* ptx_sim_kernel_info(const class function_info *kernel); - /*! * This class functionally executes a kernel. It uses the basic data structures and procedures in core_t */ diff --git a/src/cuda-sim/ptx-stats.cc b/src/cuda-sim/ptx-stats.cc index 298729f..22517df 100644 --- a/src/cuda-sim/ptx-stats.cc +++ b/src/cuda-sim/ptx-stats.cc @@ -151,27 +151,27 @@ void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn) // attribute pipeline latency to this ptx instruction (specified by the pc) // pipeline latency is the number of cycles a warp with this instruction spent in the pipeline -void ptx_file_line_stats_add_latency(unsigned pc, unsigned latency) +void ptx_stats::ptx_file_line_stats_add_latency(unsigned pc, unsigned latency) { - const ptx_instruction *pInsn = function_info::pc_to_instruction(pc); + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].latency += latency; } // attribute dram traffic to this ptx instruction (specified by the pc) // dram traffic is counted in number of requests -void ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic) +void ptx_stats::ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic) { - const ptx_instruction *pInsn = function_info::pc_to_instruction(pc); + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].dram_traffic += dram_traffic; } // attribute the number of shared memory access cycles to a ptx instruction // counts both the number of warps doing shared memory access and the number of cycles involved -void ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, unsigned n_way_bkconflict) +void ptx_stats::ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, unsigned n_way_bkconflict) { - const ptx_instruction *pInsn = function_info::pc_to_instruction(pc); + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; line_stats.smem_n_way_bank_conflict_total += n_way_bkconflict; @@ -180,9 +180,9 @@ void ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, unsigned n_way_bkco // attribute a non-coalesced mem access to a ptx instruction // counts both the number of warps causing this and the number of memory requests generated -void ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access) +void ptx_stats::ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access) { - const ptx_instruction *pInsn = function_info::pc_to_instruction(pc); + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; line_stats.gmem_n_access_total += n_access; @@ -239,17 +239,17 @@ void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores) } // add an inflight memory instruction -void ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc) +void ptx_stats::ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc) { - const ptx_instruction *pInsn = function_info::pc_to_instruction(pc); + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); inflight_mem_tracker[sc_id].add_count(pInsn); } // remove an inflight memory instruction -void ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc) +void ptx_stats::ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc) { - const ptx_instruction *pInsn = function_info::pc_to_instruction(pc); + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); inflight_mem_tracker[sc_id].sub_count(pInsn); } @@ -262,9 +262,9 @@ void ptx_file_line_stats_commit_exposed_latency(int sc_id, int exposed_latency) } // attribute the number of warp divergence to a ptx instruction -void ptx_file_line_stats_add_warp_divergence(unsigned pc, unsigned n_way_divergence) +void ptx_stats::ptx_file_line_stats_add_warp_divergence(unsigned pc, unsigned n_way_divergence) { - const ptx_instruction *pInsn = function_info::pc_to_instruction(pc); + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; line_stats.warp_divergence += n_way_divergence; diff --git a/src/cuda-sim/ptx-stats.h b/src/cuda-sim/ptx-stats.h index c75fc58..246b4ce 100644 --- a/src/cuda-sim/ptx-stats.h +++ b/src/cuda-sim/ptx-stats.h @@ -37,17 +37,10 @@ void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn); #endif // stat collection interface to gpgpu-sim -void ptx_file_line_stats_add_latency(unsigned pc, unsigned latency); -void ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic); -void ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, unsigned n_way_bkconflict); -void ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access); void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores); -void ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc); -void ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc); void ptx_file_line_stats_commit_exposed_latency(int sc_id, int exposed_latency); -void ptx_file_line_stats_add_warp_divergence(unsigned pc, unsigned n_way_divergence); class gpgpu_context; class ptx_stats { @@ -64,4 +57,12 @@ class ptx_stats { // output stats to a file void ptx_file_line_stats_write_file(); + // stat collection interface to gpgpu-sim + void ptx_file_line_stats_add_latency(unsigned pc, unsigned latency); + void ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic); + void ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, unsigned n_way_bkconflict); + void ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access); + void ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc); + void ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc); + void ptx_file_line_stats_add_warp_divergence(unsigned pc, unsigned n_way_divergence); }; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index 5fa9379..3384d49 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -43,6 +43,14 @@ typedef void * yyscan_t; #define STR_SIZE 1024 +const ptx_instruction* gpgpu_context::pc_to_instruction(unsigned pc) +{ + if( pc < s_g_pc_to_insn.size() ) + return s_g_pc_to_insn[pc]; + else + return NULL; +} + unsigned symbol::get_uid() { unsigned result = (gpgpu_ctx->symbol_sm_next_uid)++; diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 8fc0a06..f4c5c37 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1372,13 +1372,6 @@ public: return m_symtab; } - static const ptx_instruction* pc_to_instruction(unsigned pc) - { - if( pc < s_g_pc_to_insn.size() ) - return s_g_pc_to_insn[pc]; - else - return NULL; - } unsigned local_mem_framesize() const { return m_local_mem_framesize; @@ -1436,8 +1429,6 @@ private: symbol_table *m_symtab; - static std::vector s_g_pc_to_insn; // a direct mapping from PC to instruction - //parameter size for device kernels int m_args_aligned_size; diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index bdf989a..e4ae04f 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -886,7 +886,7 @@ void gpgpu_sim::init() m_shader_stats->new_grid(); // initialize the control-flow, memory access, memory latency logger if (m_config.g_visualizer_enabled) { - create_thread_CFlogger( m_config.num_shader(), m_shader_config->n_thread_per_shader, 0, m_config.gpgpu_cflog_interval ); + create_thread_CFlogger( gpgpu_ctx, m_config.num_shader(), m_shader_config->n_thread_per_shader, 0, m_config.gpgpu_cflog_interval ); } shader_CTA_count_create( m_config.num_shader(), m_config.gpgpu_cflog_interval); if (m_config.gpgpu_cflog_interval != 0) { diff --git a/src/gpgpu-sim/mem_latency_stat.cc b/src/gpgpu-sim/mem_latency_stat.cc index 4e94991..a1b43a8 100644 --- a/src/gpgpu-sim/mem_latency_stat.cc +++ b/src/gpgpu-sim/mem_latency_stat.cc @@ -41,6 +41,7 @@ #include #include #include +#include "../../libcuda/gpgpu_context.h" memory_stats_t::memory_stats_t( unsigned n_shader, const shader_core_config *shader_config, const memory_config *mem_config, const class gpgpu_sim* gpu ) { @@ -195,7 +196,7 @@ void memory_stats_t::memlatstat_dram_access(mem_fetch *mf) mem_access_type_stats[mf->get_access_type()][dram_id][bank]++; } if (mf->get_pc() != (unsigned)-1) - ptx_file_line_stats_add_dram_traffic(mf->get_pc(), mf->get_data_size()); + m_gpu->gpgpu_ctx->stats->ptx_file_line_stats_add_dram_traffic(mf->get_pc(), mf->get_data_size()); } void memory_stats_t::memlatstat_icnt2mem_pop(mem_fetch *mf) diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index b7ae95d..c697450 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -744,7 +744,7 @@ void shader_core_ctx::decode() if( m_inst_fetch_buffer.m_valid ) { // decode 1 or 2 instructions and place them into ibuffer address_type pc = m_inst_fetch_buffer.m_pc; - const warp_inst_t* pI1 = ptx_fetch_inst(pc); + const warp_inst_t* pI1 = m_gpu->gpgpu_ctx->ptx_fetch_inst(pc); m_warp[m_inst_fetch_buffer.m_warp_id].ibuffer_fill(0,pI1); m_warp[m_inst_fetch_buffer.m_warp_id].inc_inst_in_pipeline(); if( pI1 ) { @@ -754,7 +754,7 @@ void shader_core_ctx::decode() }else if(pI1->oprnd_type==FP_OP) { m_stats->m_num_FPdecoded_insn[m_sid]++; } - const warp_inst_t* pI2 = ptx_fetch_inst(pc+pI1->isize); + const warp_inst_t* pI2 = m_gpu->gpgpu_ctx->ptx_fetch_inst(pc+pI1->isize); if( pI2 ) { m_warp[m_inst_fetch_buffer.m_warp_id].ibuffer_fill(1,pI2); m_warp[m_inst_fetch_buffer.m_warp_id].inc_inst_in_pipeline(); diff --git a/src/gpgpu-sim/stat-tool.cc b/src/gpgpu-sim/stat-tool.cc index 6a4c75b..35a4cc3 100644 --- a/src/gpgpu-sim/stat-tool.cc +++ b/src/gpgpu-sim/stat-tool.cc @@ -37,6 +37,7 @@ #include #include #include +#include "../../libcuda/gpgpu_context.h" //////////////////////////////////////////////////////////////////////////////// @@ -110,12 +111,10 @@ void spill_log_to_file (FILE *fout, int final, unsigned long long current_cycle //////////////////////////////////////////////////////////////////////////////// -unsigned translate_pc_to_ptxlineno(unsigned pc); - static int n_thread_CFloggers = 0; static thread_CFlocality** thread_CFlogger = NULL; -void create_thread_CFlogger( int n_loggers, int n_threads, address_type start_pc, unsigned long long logging_interval) +void create_thread_CFlogger(gpgpu_context* ctx, int n_loggers, int n_threads, address_type start_pc, unsigned long long logging_interval) { destroy_thread_CFlogger(); @@ -126,7 +125,7 @@ void create_thread_CFlogger( int n_loggers, int n_threads, address_type start_pc char buffer[32]; for (int i = 0; i < n_thread_CFloggers; i++) { snprintf(buffer, 32, "%02d", i); - thread_CFlogger[i] = new thread_CFlocality( name_tpl + buffer, logging_interval, n_threads, start_pc); + thread_CFlogger[i] = new thread_CFlocality( ctx, name_tpl + buffer, logging_interval, n_threads, start_pc); if (logging_interval != 0) { add_snap_shot_trigger(thread_CFlogger[i]); add_spill_log(thread_CFlogger[i]); @@ -368,10 +367,10 @@ static int s_cache_access_logger_n_types = 0; static std::vector s_cache_access_logger; enum cache_access_logger_types { - NORMAL, TEXTURE, CONSTANT, INSTRUCTION + NORMALS, TEXTURE, CONSTANT, INSTRUCTION }; -int get_shader_normal_cache_id() { return NORMAL; } +int get_shader_normal_cache_id() { return NORMALS; } int get_shader_texture_cache_id() { return TEXTURE; } int get_shader_constant_cache_id() { return CONSTANT; } int get_shader_instruction_cache_id() { return INSTRUCTION; } @@ -394,7 +393,7 @@ void shader_cache_access_log( int logger_id, int type, int miss) { if (s_cache_access_logger_n_types == 0) return; if (logger_id < 0) return; - assert(type == NORMAL || type == TEXTURE || type == CONSTANT || type == INSTRUCTION); + assert(type == NORMALS || type == TEXTURE || type == CONSTANT || type == INSTRUCTION); assert(miss == 0 || miss == 1); s_cache_access_logger[logger_id].log(2 * type + miss); @@ -404,7 +403,7 @@ void shader_cache_access_unlog( int logger_id, int type, int miss) { if (s_cache_access_logger_n_types == 0) return; if (logger_id < 0) return; - assert(type == NORMAL || type == TEXTURE || type == CONSTANT || type == INSTRUCTION); + assert(type == NORMALS || type == TEXTURE || type == CONSTANT || type == INSTRUCTION); assert(miss == 0 || miss == 1); s_cache_access_logger[logger_id].unlog(2 * type + miss); @@ -477,22 +476,24 @@ void shader_CTA_count_visualizer_gzprint( gzFile fout ) //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// -thread_insn_span::thread_insn_span(unsigned long long cycle) +thread_insn_span::thread_insn_span(unsigned long long cycle, gpgpu_context* ctx) : m_cycle(cycle), #if (tr1_hash_map_ismap == 1) m_insn_span_count() #else m_insn_span_count(32*1024) #endif -{ +{ + gpgpu_ctx = ctx; } thread_insn_span::~thread_insn_span() { } -thread_insn_span::thread_insn_span(const thread_insn_span& other) +thread_insn_span::thread_insn_span(const thread_insn_span& other, gpgpu_context* ctx) : m_cycle(other.m_cycle), - m_insn_span_count(other.m_insn_span_count) + m_insn_span_count(other.m_insn_span_count) { + gpgpu_ctx = ctx; } thread_insn_span& thread_insn_span::operator=(const thread_insn_span& other) @@ -551,7 +552,7 @@ void thread_insn_span::print_sparse_histo(FILE *fout) const int n_printed_entries = 0; span_count_map::const_iterator i_sc = m_insn_span_count.begin(); for (; i_sc != m_insn_span_count.end(); ++i_sc) { - unsigned ptx_lineno = translate_pc_to_ptxlineno(i_sc->first); + unsigned ptx_lineno = gpgpu_ctx->translate_pc_to_ptxlineno(i_sc->first); fprintf(fout, "%u %d ", ptx_lineno, i_sc->second); n_printed_entries++; } @@ -566,7 +567,7 @@ void thread_insn_span::print_sparse_histo(gzFile fout) const int n_printed_entries = 0; span_count_map::const_iterator i_sc = m_insn_span_count.begin(); for (; i_sc != m_insn_span_count.end(); ++i_sc) { - unsigned ptx_lineno = translate_pc_to_ptxlineno(i_sc->first); + unsigned ptx_lineno = gpgpu_ctx->translate_pc_to_ptxlineno(i_sc->first); gzprintf(fout, "%u %d ", ptx_lineno, i_sc->second); n_printed_entries++; } @@ -578,14 +579,14 @@ void thread_insn_span::print_sparse_histo(gzFile fout) const //////////////////////////////////////////////////////////////////////////////// -thread_CFlocality::thread_CFlocality(std::string name, +thread_CFlocality::thread_CFlocality( gpgpu_context* ctx, std::string name, unsigned long long snap_shot_interval, int nthreads, address_type start_pc, unsigned long long start_cycle) : snap_shot_trigger(snap_shot_interval), m_name(name), m_nthreads(nthreads), m_thread_pc(nthreads, start_pc), m_cycle(start_cycle), - m_thd_span(start_cycle) + m_thd_span(start_cycle, ctx) { std::fill(m_thread_pc.begin(), m_thread_pc.end(), -1); // so that hw thread with no work assigned will not clobber results } diff --git a/src/gpgpu-sim/stat-tool.h b/src/gpgpu-sim/stat-tool.h index 5646f01..67b3923 100644 --- a/src/gpgpu-sim/stat-tool.h +++ b/src/gpgpu-sim/stat-tool.h @@ -35,6 +35,7 @@ #include #include +class gpgpu_context; ///////////////////////////////////////////////////////////////////////////////////// // logger snapshot trigger: // - automate the snap_shot part of loggers to avoid modifying simulation loop everytime @@ -80,8 +81,8 @@ public: class thread_insn_span { public: - thread_insn_span(unsigned long long cycle); - thread_insn_span(const thread_insn_span& other); + thread_insn_span(unsigned long long cycle, gpgpu_context* ctx); + thread_insn_span(const thread_insn_span& other, gpgpu_context* ctx); ~thread_insn_span(); thread_insn_span& operator=(const thread_insn_span& other); @@ -94,7 +95,8 @@ public: void print_sparse_histo(FILE *fout) const; void print_sparse_histo(gzFile fout) const; -private: +private: + gpgpu_context* gpgpu_ctx; typedef tr1_hash_map span_count_map; unsigned long long m_cycle; span_count_map m_insn_span_count; @@ -102,7 +104,7 @@ private: class thread_CFlocality : public snap_shot_trigger, public spill_log_interface { public: - thread_CFlocality(std::string name, unsigned long long snap_shot_interval, + thread_CFlocality(gpgpu_context* ctx, std::string name, unsigned long long snap_shot_interval, int nthreads, address_type start_pc, unsigned long long start_cycle = 0); ~thread_CFlocality(); @@ -270,7 +272,7 @@ void try_snap_shot (unsigned long long current_cycle); void set_spill_interval (unsigned long long interval); void spill_log_to_file (FILE *fout, int final, unsigned long long current_cycle); -void create_thread_CFlogger( int n_loggers, int n_threads, address_type start_pc, unsigned long long logging_interval); +void create_thread_CFlogger(gpgpu_context* ctx, int n_loggers, int n_threads, address_type start_pc, unsigned long long logging_interval); void destroy_thread_CFlogger( ); void cflog_update_thread_pc( int logger_id, int thread_id, address_type pc ); void cflog_snapshot( int logger_id, unsigned long long cycle ); -- cgit v1.3 From eb3b077cddb24a734bb72f5c1588dee4034214a7 Mon Sep 17 00:00:00 2001 From: Mahmoud Date: Wed, 4 Sep 2019 21:38:57 -0400 Subject: adding trace class and parss kernel info --- Makefile | 4 +- libcuda/gpgpu_context.h | 1 + src/cuda-sim/ptx_ir.h | 10 +- src/cuda-sim/ptx_loader.cc | 2 +- src/cuda-sim/ptx_parser.cc | 4 +- src/gpgpu-sim/gpu-sim.cc | 9 + src/gpgpu-sim/gpu-sim.h | 7 + src/gpgpusim_entrypoint.cc | 32 ++- src/gpgpusim_entrypoint.h | 2 + src/trace-driven/gpgpusim_trace_driven_main.cc | 305 +++++++++++++++---------- src/trace-driven/trace_driven.h | 71 ++++++ 11 files changed, 322 insertions(+), 125 deletions(-) create mode 100644 src/trace-driven/trace_driven.h (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/Makefile b/Makefile index caf0a17..5cd3101 100644 --- a/Makefile +++ b/Makefile @@ -144,7 +144,7 @@ no_opencl_support: @echo "Warning: gpgpu-sim is building without opencl support. Make sure NVOPENCL_LIBDIR and NVOPENCL_INCDIR are set" $(SIM_LIB_DIR)/libcudart.so: makedirs $(LIBS) cudalib - g++ -shared -Wl,-soname,libcudart_$(GPGPUSIM_BUILD).so -Wl,--version-script=linux-so-version.txt\ + g++ -shared -Wl,-soname,libcudart.so -Wl,--version-script=linux-so-version.txt\ $(SIM_OBJ_FILES_DIR)/libcuda/*.o \ $(SIM_OBJ_FILES_DIR)/cuda-sim/*.o \ $(SIM_OBJ_FILES_DIR)/cuda-sim/decuda_pred_table/*.o \ @@ -171,7 +171,7 @@ $(SIM_LIB_DIR)/libcudart.so: makedirs $(LIBS) cudalib $(SIM_LIB_DIR)/gpgpusim.out: makedirs $(LIBS) cudalib $(SIM_LIB_DIR)/libcudart.so - g++ -std=c++0x -L$(SIM_LIB_DIR) -lcudart -o $(SIM_LIB_DIR)/gpgpusim.out src/trace-driven/gpgpusim_trace_driven_main.cc + g++ -std=c++0x -L$(SIM_LIB_DIR) -I$(CUDA_INSTALL_PATH)/include -lcudart -o $(SIM_LIB_DIR)/gpgpusim.out src/trace-driven/gpgpusim_trace_driven_main.cc $(SIM_LIB_DIR)/libcudart.dylib: makedirs $(LIBS) cudalib g++ -dynamiclib -Wl,-headerpad_max_install_names,-undefined,dynamic_lookup,-compatibility_version,1.1,-current_version,1.1\ diff --git a/libcuda/gpgpu_context.h b/libcuda/gpgpu_context.h index 45c5cdd..d3c5d74 100644 --- a/libcuda/gpgpu_context.h +++ b/libcuda/gpgpu_context.h @@ -65,6 +65,7 @@ class gpgpu_context { const ptx_instruction* pc_to_instruction(unsigned pc); const warp_inst_t *ptx_fetch_inst( address_type pc ); unsigned translate_pc_to_ptxlineno(unsigned pc); + class gpgpu_sim *gpgpu_trace_sim_init_perf(int argc, const char *argv[]); }; gpgpu_context* GPGPU_Context(); diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index f4c5c37..8ceb60b 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1354,13 +1354,13 @@ public: void list_param( FILE *fout ) const; void ptx_jit_config(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) ; - const struct gpgpu_ptx_sim_info* get_kernel_info () const + virtual const struct gpgpu_ptx_sim_info* get_kernel_info () const { assert (m_kernel_info.maxthreads == maxnt_id); return &m_kernel_info; } - const void set_kernel_info (const struct gpgpu_ptx_sim_info &info) { + virtual const void set_kernel_info (const struct gpgpu_ptx_sim_info &info) { m_kernel_info = info; m_kernel_info.ptx_version = 10*get_ptx_version().ver(); m_kernel_info.sm_target = get_ptx_version().target(); @@ -1401,6 +1401,10 @@ public: // backward pointer class gpgpu_context* gpgpu_ctx; +protected: + //Registers/shmem/etc. used (from ptxas -v), loaded from ___.ptxinfo along with ___.ptx + struct gpgpu_ptx_sim_info m_kernel_info; + private: unsigned maxnt_id; unsigned m_uid; @@ -1424,8 +1428,6 @@ private: std::map labels; unsigned num_reconvergence_pairs; - //Registers/shmem/etc. used (from ptxas -v), loaded from ___.ptxinfo along with ___.ptx - struct gpgpu_ptx_sim_info m_kernel_info; symbol_table *m_symtab; diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc index dca3cec..207fa1f 100644 --- a/src/cuda-sim/ptx_loader.cc +++ b/src/cuda-sim/ptx_loader.cc @@ -37,7 +37,7 @@ /// extern prototypes -extern int ptx_error( yyscan_t yyscanner, const char *s ); +extern int ptx_error( yyscan_t yyscanner, ptx_recognizer* recognizer, const char *s ); extern int ptx_lex_init(yyscan_t* scanner); extern void ptx_set_in(FILE * _in_str ,yyscan_t yyscanner ); extern int ptx_parse(yyscan_t scanner, ptx_recognizer* recognizer); diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index 81b70af..9e9d500 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -35,7 +35,7 @@ typedef void * yyscan_t; extern int ptx_get_lineno (yyscan_t yyscanner ); extern YYSTYPE* ptx_get_lval (yyscan_t yyscanner ); -extern int ptx_error( yyscan_t yyscanner, const char *s ); +extern int ptx_error( yyscan_t yyscanner, ptx_recognizer* recognizer, const char *s ); extern int ptx_lex_init(yyscan_t* scanner); extern void ptx_set_in(FILE * _in_str ,yyscan_t yyscanner ); extern FILE *ptx_get_in (yyscan_t yyscanner ); @@ -226,7 +226,7 @@ void ptx_recognizer::parse_error_impl( const char *file, unsigned line, const ch g_error_detected = 1; printf("%s:%u: Parse error: %s (%s:%u)\n\n", gpgpu_ctx->g_filename, ptx_get_lineno(scanner), buf, file, line); - ptx_error(scanner, NULL); + ptx_error(scanner, this, NULL); abort(); exit(1); } diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index a151379..234a8d6 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -560,6 +560,15 @@ void gpgpu_sim_config::reg_options(option_parser_t opp) option_parser_register(opp, "-gpgpu_cdp_enabled", OPT_BOOL, &(gpgpu_ctx->device_runtime->g_cdp_enabled), "Turn on CDP", "0"); + + //Trace driven mode parameters + option_parser_register(opp, "-trace_driven_mode", OPT_BOOL, + &trace_driven_mode, "Turn on trace_driven_mode", + "0"); + option_parser_register(opp, "-trace", OPT_CSTR, + &g_traces_filename, "traces kernel file" + "traces kernel file directory", + "./traces/kernelslist.g"); } ///////////////////////////////////////////////////////////////////////////// diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h index 76c7a06..6b57bd8 100644 --- a/src/gpgpu-sim/gpu-sim.h +++ b/src/gpgpu-sim/gpu-sim.h @@ -348,6 +348,9 @@ public: size_t sync_depth_limit() const {return runtime_sync_depth_limit; } size_t pending_launch_count_limit() const {return runtime_pending_launch_count_limit;} + unsigned is_trace_driven_mode() const { return trace_driven_mode; } + char* get_traces_filename() const { return g_traces_filename; } + private: void init_clock_domains(void ); @@ -401,6 +404,10 @@ private: unsigned int gpgpu_compute_capability_minor; unsigned long long liveness_message_freq; + //trace driven mode options + bool trace_driven_mode; + char *g_traces_filename; + friend class gpgpu_sim; }; diff --git a/src/gpgpusim_entrypoint.cc b/src/gpgpusim_entrypoint.cc index 816159f..4da74c0 100644 --- a/src/gpgpusim_entrypoint.cc +++ b/src/gpgpusim_entrypoint.cc @@ -60,7 +60,6 @@ class stream_manager* g_stream_manager() { return GPGPUsim_ctx_ptr()->g_stream_manager; } -static void print_simulation_time(); void *gpgpu_sim_thread_sequential(void*) { @@ -242,6 +241,37 @@ gpgpu_sim *gpgpu_context::gpgpu_ptx_sim_init_perf() return GPGPUsim_ctx_ptr()->g_the_gpu; } +gpgpu_sim *gpgpu_context::gpgpu_trace_sim_init_perf(int argc, const char *argv[]) +{ + srand(1); + print_splash(); + func_sim->read_sim_environment_variables(); + ptx_parser->read_parser_environment_variables(); + option_parser_t opp = option_parser_create(); + + ptx_reg_options(opp); + func_sim->ptx_opcocde_latency_options(opp); + + icnt_reg_options(opp); + GPGPUsim_ctx_ptr()->g_the_gpu_config = new gpgpu_sim_config(this); + GPGPUsim_ctx_ptr()->g_the_gpu_config->reg_options(opp); // register GPU microrachitecture options + + option_parser_cmdline(opp, argc, argv); // parse configuration options + fprintf(stdout, "GPGPU-Sim: Configuration options:\n\n"); + option_parser_print(opp, stdout); + // Set the Numeric locale to a standard locale where a decimal point is a "dot" not a "comma" + // so it does the parsing correctly independent of the system environment variables + assert(setlocale(LC_NUMERIC,"C")); + GPGPUsim_ctx_ptr()->g_the_gpu_config->init(); + + GPGPUsim_ctx_ptr()->g_the_gpu = new gpgpu_sim(*(GPGPUsim_ctx_ptr()->g_the_gpu_config), this); + GPGPUsim_ctx_ptr()->g_stream_manager = new stream_manager((GPGPUsim_ctx_ptr()->g_the_gpu), func_sim->g_cuda_launch_blocking); + + GPGPUsim_ctx_ptr()->g_simulation_starttime = time((time_t *)NULL); + + return GPGPUsim_ctx_ptr()->g_the_gpu; +} + void start_sim_thread(int api) { if( GPGPUsim_ctx_ptr()->g_sim_done ) { diff --git a/src/gpgpusim_entrypoint.h b/src/gpgpusim_entrypoint.h index 887b3c8..4a8d796 100644 --- a/src/gpgpusim_entrypoint.h +++ b/src/gpgpusim_entrypoint.h @@ -84,4 +84,6 @@ class stream_manager* g_stream_manager(); int gpgpu_opencl_ptx_sim_main_perf( kernel_info_t *grid ); +void print_simulation_time(); + #endif diff --git a/src/trace-driven/gpgpusim_trace_driven_main.cc b/src/trace-driven/gpgpusim_trace_driven_main.cc index 6553fd7..881e351 100644 --- a/src/trace-driven/gpgpusim_trace_driven_main.cc +++ b/src/trace-driven/gpgpusim_trace_driven_main.cc @@ -1,138 +1,213 @@ //developed by Mahmoud Khairy, Purdue Univ //abdallm@purdue.edu -#include "../abstract_hardware_model.h" +//#include "../abstract_hardware_model.h" #include #include +#include +#include +#include +#include +#include +#include +#include + -#include "../option_parser.h" -#include "../cuda-sim/cuda-sim.h" -#include "../cuda-sim/ptx_ir.h" -#include "../cuda-sim/ptx_parser.h" +//#include "../option_parser.h" +//#include "../cuda-sim/cuda-sim.h" +//#include "../cuda-sim/ptx_ir.h" +//#include "../cuda-sim/ptx_parser.h" #include "../gpgpu-sim/gpu-sim.h" -#include "../gpgpu-sim/icnt_wrapper.h" +//#include "../gpgpu-sim/icnt_wrapper.h" +//#include "../gpgpu-sim/icnt_wrapper.h" +#include "../../libcuda/gpgpu_context.h" +#include "trace_driven.h" + //#include "../stream_manager.h" +void arguments_check(); + +int main ( int argc, const char **argv ) +{ + + gpgpu_context* m_gpgpu_context = GPGPU_Context(); + gpgpu_sim * m_gpgpu_sim = m_gpgpu_context->gpgpu_trace_sim_init_perf(argc,argv); + m_gpgpu_sim->init(); + + //for each kernel + //load file + //parse and create kernel info + //launch + //while loop till the end of the end kernel execution + //prints stats + + trace_parser tracer(m_gpgpu_sim->get_config().get_traces_filename(), m_gpgpu_sim, m_gpgpu_context); + + std::vector kernellist; + tracer.parse_kernellist_file(kernellist); + + for(unsigned i=0; ilaunch(kernel_info); + + bool active = false; + bool sim_cycles = false; + bool break_limit = false; + + do { + if(!m_gpgpu_sim->active()) + break; + + //performance simulation + if( m_gpgpu_sim->active() ) { + m_gpgpu_sim->cycle(); + sim_cycles = true; + m_gpgpu_sim->deadlock_check(); + }else { + if(m_gpgpu_sim->cycle_insn_cta_max_hit()){ + g_stream_manager()->stop_all_running_kernels(); + break_limit = true; + } + } + + active=m_gpgpu_sim->active() ; + + } while( active ); -gpgpu_sim_config g_the_gpu_config; -gpgpu_sim *g_the_gpu; -time_t g_simulation_starttime; + tracer.kernel_finalizer(kernel_info); -#define MAX(a,b) (((a)>(b))?(a):(b)) + m_gpgpu_sim->print_stats(); -static void print_simulation_time(); + if(sim_cycles) { + m_gpgpu_sim->update_stats(); + print_simulation_time(); + } + if(break_limit) { + printf("GPGPU-Sim: ** break due to reaching the maximum cycles (or instructions) **\n"); + fflush(stdout); + exit(1); + } + } -int main ( int argc, char **argv ) + return 1; +} + +trace_parser::trace_parser(const char* kernellist_filepath, gpgpu_sim * m_gpgpu_sim, gpgpu_context* m_gpgpu_context) { - srand(1); - - option_parser_t opp = option_parser_create(); - - icnt_reg_options(opp); - g_the_gpu_config.reg_options(opp); // register GPU microrachitecture options - ptx_reg_options(opp); - ptx_opcocde_latency_options(opp); - - //ptx_opcocde_latency_options(opp); //do this for trace driven - - fprintf(stdout, "I am here:\n\n"); - option_parser_cmdline(opp, argc, (const char **)argv); // parse configuration options - - fprintf(stdout, "GPGPU-Sim: Configuration options:\n\n"); - option_parser_print(opp, stdout); - // Set the Numeric locale to a standard locale where a decimal point is a "dot" not a "comma" - // so it does the parsing correctly independent of the system environment variables - assert(setlocale(LC_NUMERIC,"C")); - g_the_gpu_config.init(); - - g_the_gpu = new gpgpu_sim(g_the_gpu_config); - //g_stream_manager = new stream_manager(g_the_gpu,g_cuda_launch_blocking); - - //load file - //create kernel info - //launch - //while loop till the end - //prints stats - //g_the_gpu->launch(grid); - - g_simulation_starttime = time((time_t *)NULL); - - bool active = false; - bool sim_cycles = false; - bool break_limit = false; - - g_the_gpu->init(); - do { - // check if a kernel has completed - // launch operation on device if one is pending and can be run - - // Need to break this loop when a kernel completes. This was a - // source of non-deterministic behaviour in GPGPU-Sim (bug 147). - // If another stream operation is available, g_the_gpu remains active, - // causing this loop to not break. If the next operation happens to be - // another kernel, the gpu is not re-initialized and the inter-kernel - // behaviour may be incorrect. Check that a kernel has finished and - // no other kernel is currently running. - if(!g_the_gpu->active()) - break; - - //performance simulation - if( g_the_gpu->active() ) { - g_the_gpu->cycle(); - sim_cycles = true; - g_the_gpu->deadlock_check(); - }else { - if(g_the_gpu->cycle_insn_cta_max_hit()){ - g_stream_manager->stop_all_running_kernels(); - break_limit = true; - } - } - - active=g_the_gpu->active() ; - - } while( active ); - - printf("GPGPU-Sim: ** STOP simulation thread (no work) **\n"); - fflush(stdout); - - g_the_gpu->print_stats(); - - if(sim_cycles) { - g_the_gpu->update_stats(); - print_simulation_time(); - } - - - printf("GPGPU-Sim: *** simulation thread exiting ***\n"); - fflush(stdout); - - if(break_limit) { - printf("GPGPU-Sim: ** break due to reaching the maximum cycles (or instructions) **\n"); + + this->m_gpgpu_sim = m_gpgpu_sim; + this->m_gpgpu_context = m_gpgpu_context; + kernellist_filename = kernellist_filepath; +} + +void trace_parser::parse_kernellist_file(std::vector& kernellist) { + + ifs.open(kernellist_filename); + + if (!ifs.is_open()) { + std::cout << "Unable to open file: " <>string1>>string2; + if(string1 == "kernel" && string2 == "name") { + const size_t equal_idx = line.find('='); + kernel_name = line.substr(equal_idx+1); + } + else if(string1 == "kernel" && string2 == "id") { + sscanf(line.c_str(), "-kernel id = %d", &kernel_id); + } + else if(string1 == "grid" && string2 == "dim") { + sscanf(line.c_str(), "-grid dim = (%d,%d,%d)", &grid_dim_x, &grid_dim_y, &grid_dim_z); + } + else if (string1 == "block" && string2 == "dim") { + sscanf(line.c_str(), "-block dim = (%d,%d,%d)", &tb_dim_x, &tb_dim_y, &tb_dim_z); + } + else if (string1 == "shmem") { + sscanf(line.c_str(), "-shmem = %d", &shmem); + } + else if (string1 == "nregs") { + sscanf(line.c_str(), "-nregs = %d", &nregs); + } + else if (string1 == "cuda" && string2 == "stream") { + sscanf(line.c_str(), "-cuda stream id = %d", &cuda_stream_id); + } + continue; + } + } + + gpgpu_ptx_sim_info info; + info.smem = shmem; + info.regs = nregs; + dim3 gridDim(grid_dim_x, grid_dim_y, grid_dim_z); + dim3 blockDim(tb_dim_x, tb_dim_y, tb_dim_z); + trace_function_info* function_info = new trace_function_info(info, m_gpgpu_context); + trace_kernel_info_t* kernel_info = new trace_kernel_info_t(gridDim, blockDim, function_info, kerneltraces_filepath); + + return kernel_info; } -void print_simulation_time() -{ - time_t current_time, difference, d, h, m, s; - current_time = time((time_t *)NULL); - difference = MAX(current_time - g_simulation_starttime, 1); - - d = difference/(3600*24); - h = difference/3600 - 24*d; - m = difference/60 - 60*(h + 24*d); - s = difference - 60*(m + 60*(h + 24*d)); - - fflush(stderr); - printf("\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", - (unsigned)d, (unsigned)h, (unsigned)m, (unsigned)s, (unsigned)difference ); - printf("gpgpu_simulation_rate = %u (inst/sec)\n", (unsigned)(g_the_gpu->gpu_tot_sim_insn / difference) ); - printf("gpgpu_simulation_rate = %u (cycle/sec)\n", (unsigned)(gpu_tot_sim_cycle / difference) ); - fflush(stdout); + +void trace_parser::kernel_finalizer(trace_kernel_info_t* kernel_info){ + if (ifs.is_open()) + ifs.close(); + + delete kernel_info->entry(); + delete kernel_info; } + diff --git a/src/trace-driven/trace_driven.h b/src/trace-driven/trace_driven.h new file mode 100644 index 0000000..91ff90b --- /dev/null +++ b/src/trace-driven/trace_driven.h @@ -0,0 +1,71 @@ +//developed by Mahmoud Khairy, Purdue Univ + +#include +#include +#include + +#ifndef TRACE_DRIVEN_H +#define TRACE_DRIVEN_H + +#include "../abstract_hardware_model.h" + + +class trace_function_info: public function_info { +public: + trace_function_info(const struct gpgpu_ptx_sim_info &info, gpgpu_context* m_gpgpu_context):function_info(0, m_gpgpu_context ) { + m_kernel_info = info; + } + + virtual const struct gpgpu_ptx_sim_info* get_kernel_info () const { + return &m_kernel_info; + } + + virtual const void set_kernel_info (const struct gpgpu_ptx_sim_info &info) { + m_kernel_info = info; + } + +private: + + +}; + +class trace_kernel_info_t: public kernel_info_t { +public: + trace_kernel_info_t(dim3 gridDim, dim3 blockDim, trace_function_info* m_function_info, const std::string& filepath):kernel_info_t(gridDim, blockDim, m_function_info) { + traces_filepath = filepath; + } + +private: + std::string traces_filepath; + +}; + +class trace_warp_inst_t: public warp_inst_t { +public: + + trace_warp_inst_t() { + } + +private: + + +}; + +class trace_parser { +public: + trace_parser(const char* kernellist_filepath, gpgpu_sim * m_gpgpu_sim, gpgpu_context* m_gpgpu_context); + + void parse_kernellist_file(std::vector& kernellist); + trace_kernel_info_t* parse_kernel_info(const std::string& kerneltraces_filepath); + void kernel_finalizer(trace_kernel_info_t* kernel_info); + +private: + + std::string kernellist_filename; + std::ifstream ifs; + gpgpu_sim * m_gpgpu_sim; + gpgpu_context* m_gpgpu_context; + +}; + +#endif -- cgit v1.3 From 0c023e41809dba8897c37af6bb03e5c3aa9ebc5e Mon Sep 17 00:00:00 2001 From: Nick Date: Fri, 13 Sep 2019 05:32:48 -0400 Subject: Add src/cuda-sim formatting --- src/cuda-sim/cuda-math.h | 455 +- src/cuda-sim/cuda-sim.cc | 4899 +++++++-------- src/cuda-sim/cuda-sim.h | 287 +- src/cuda-sim/cuda_device_printf.cc | 165 +- src/cuda-sim/cuda_device_printf.h | 10 +- src/cuda-sim/cuda_device_runtime.cc | 570 +- src/cuda-sim/cuda_device_runtime.h | 91 +- src/cuda-sim/half.h | 6539 +++++++++++--------- src/cuda-sim/instructions.cc | 10840 ++++++++++++++++++---------------- src/cuda-sim/memory.cc | 336 +- src/cuda-sim/memory.h | 158 +- src/cuda-sim/opcodes.h | 77 +- src/cuda-sim/ptx-stats.cc | 376 +- src/cuda-sim/ptx-stats.h | 54 +- src/cuda-sim/ptx_ir.cc | 2665 +++++---- src/cuda-sim/ptx_ir.h | 2947 +++++---- src/cuda-sim/ptx_loader.cc | 951 +-- src/cuda-sim/ptx_loader.h | 38 +- src/cuda-sim/ptx_parser.cc | 1785 +++--- src/cuda-sim/ptx_parser.h | 302 +- src/cuda-sim/ptx_sim.cc | 1061 ++-- src/cuda-sim/ptx_sim.h | 912 ++- 22 files changed, 18790 insertions(+), 16728 deletions(-) (limited to 'src/cuda-sim/ptx_ir.h') diff --git a/src/cuda-sim/cuda-math.h b/src/cuda-sim/cuda-math.h index 9a5468c..97ce130 100644 --- a/src/cuda-sim/cuda-math.h +++ b/src/cuda-sim/cuda-math.h @@ -1,6 +1,6 @@ // This file created from vector_types.h distributed with CUDA 1.1 // (see original copyright notice below) -// +// // Changes Copyright (c) 2009-2011, Tor M. Aamodt, Wilson W.L. Fung // The University of British Columbia // All rights reserved. @@ -10,14 +10,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -28,43 +30,41 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - /* * Copyright 1993-2007 NVIDIA Corporation. All rights reserved. * - * NOTICE TO USER: + * NOTICE TO USER: * - * This source code is subject to NVIDIA ownership rights under U.S. and - * international Copyright laws. Users and possessors of this source code - * are hereby granted a nonexclusive, royalty-free license to use this code + * This source code is subject to NVIDIA ownership rights under U.S. and + * international Copyright laws. Users and possessors of this source code + * are hereby granted a nonexclusive, royalty-free license to use this code * in individual and commercial software. * - * NVIDIA MAKES NO REPRESENTATION ABOUT THE SUITABILITY OF THIS SOURCE - * CODE FOR ANY PURPOSE. IT IS PROVIDED "AS IS" WITHOUT EXPRESS OR - * IMPLIED WARRANTY OF ANY KIND. NVIDIA DISCLAIMS ALL WARRANTIES WITH - * REGARD TO THIS SOURCE CODE, INCLUDING ALL IMPLIED WARRANTIES OF + * NVIDIA MAKES NO REPRESENTATION ABOUT THE SUITABILITY OF THIS SOURCE + * CODE FOR ANY PURPOSE. IT IS PROVIDED "AS IS" WITHOUT EXPRESS OR + * IMPLIED WARRANTY OF ANY KIND. NVIDIA DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOURCE CODE, INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY, NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. - * IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL, - * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS - * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE - * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE - * OR PERFORMANCE OF THIS SOURCE CODE. + * IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL, + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE + * OR PERFORMANCE OF THIS SOURCE CODE. * - * U.S. Government End Users. This source code is a "commercial item" as - * that term is defined at 48 C.F.R. 2.101 (OCT 1995), consisting of - * "commercial computer software" and "commercial computer software - * documentation" as such terms are used in 48 C.F.R. 12.212 (SEPT 1995) - * and is provided to the U.S. Government only as a commercial end item. - * Consistent with 48 C.F.R.12.212 and 48 C.F.R. 227.7202-1 through - * 227.7202-4 (JUNE 1995), all U.S. Government End Users acquire the - * source code with only those rights set forth herein. + * U.S. Government End Users. This source code is a "commercial item" as + * that term is defined at 48 C.F.R. 2.101 (OCT 1995), consisting of + * "commercial computer software" and "commercial computer software + * documentation" as such terms are used in 48 C.F.R. 12.212 (SEPT 1995) + * and is provided to the U.S. Government only as a commercial end item. + * Consistent with 48 C.F.R.12.212 and 48 C.F.R. 227.7202-1 through + * 227.7202-4 (JUNE 1995), all U.S. Government End Users acquire the + * source code with only those rights set forth herein. * - * Any use of this source code in individual and commercial software must + * Any use of this source code in individual and commercial software must * include, in the user documentation and internal comments to the code, * the above Disclaimer and U.S. Government End Users Notice. */ - #ifndef CUDA_MATH #define CUDA_MATH @@ -74,32 +74,31 @@ #undef max #undef min namespace cuda_math { -#define __attribute__(a) // to remove warnings inside math_functions.h +#define __attribute__(a) // to remove warnings inside math_functions.h #undef INT_MAX #if CUDART_VERSION < 3000 // DEVICE_BUILTIN - struct int4 { - int x, y, z, w; - }; - struct uint4 { - unsigned int x, y, z, w; - }; - struct float4 { - float x, y, z, w; - }; - struct float2 { - float x, y; - }; - +struct int4 { + int x, y, z, w; +}; +struct uint4 { + unsigned int x, y, z, w; +}; +struct float4 { + float x, y, z, w; +}; +struct float2 { + float x, y; +}; // DEVICE_BUILTIN - typedef struct int4 int4; - typedef struct uint4 uint4; - typedef struct float4 float4; - typedef struct float2 float2; +typedef struct int4 int4; +typedef struct uint4 uint4; +typedef struct float4 float4; +typedef struct float2 float2; -extern float rsqrtf(float); // CUDA 2.3 beta +extern float rsqrtf(float); // CUDA 2.3 beta #define CUDA_FLOAT_MATH_FUNCTIONS #include @@ -108,38 +107,36 @@ extern float rsqrtf(float); // CUDA 2.3 beta #undef __CUDA_INTERNAL_COMPILATION__ #undef __attribute__ -// float to integer conversion -int float2int(float a, enum cudaRoundMode mode) -{ - return __internal_float2uint(a, mode); +// float to integer conversion +int float2int(float a, enum cudaRoundMode mode) { + return __internal_float2uint(a, mode); } -// float to unsigned integer conversion -unsigned int float2uint(float a, enum cudaRoundMode mode) -{ - return __internal_float2uint(a, mode); +// float to unsigned integer conversion +unsigned int float2uint(float a, enum cudaRoundMode mode) { + return __internal_float2uint(a, mode); } float __ll2float_rz(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_ru(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rd(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } #else @@ -147,205 +144,211 @@ float __ll2float_rd(long long int a) { #define CUDA_FLOAT_MATH_FUNCTIONS #define __CUDACC__ -// implementing int to float intrinsics with different rounding modes +// implementing int to float intrinsics with different rounding modes #include #include - // 32-bit integer to float float __int2float_rn(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_rz(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_ru(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_rd(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } // 32-bit unsigned integer to float float __uint2float_rn(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_rz(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_ru(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_rd(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } // 64-bit integer to float float __ll2float_rn(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rz(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_ru(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rd(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } -// 64-bit unsigned integer to float +// 64-bit unsigned integer to float float __ull2float_rn(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_rz(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_ru(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_rd(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; -} - -// float to integer conversion -int float2int(float a, enum cudaRoundMode mode) -{ - int tmp; - switch (mode) { - case cudaRoundZero: tmp = truncf(a); break; - case cudaRoundNearest: tmp = nearbyintf(a); break; - case cudaRoundMinInf: tmp = floorf(a); break; - case cudaRoundPosInf: tmp = ceilf(a); break; - default: abort(); - } - return tmp; -} - -int __internal_float2int(float a, enum cudaRoundMode mode) -{ - return float2int(a, mode); -} - -// float to unsigned integer conversion -unsigned int float2uint(float a, enum cudaRoundMode mode) -{ - unsigned int tmp; - switch (mode) { - case cudaRoundZero: tmp = truncf(a); break; - case cudaRoundNearest: tmp = nearbyintf(a); break; - case cudaRoundMinInf: tmp = floorf(a); break; - case cudaRoundPosInf: tmp = ceilf(a); break; - default: abort(); - } - return tmp; -} - -unsigned int __internal_float2uint(float a, enum cudaRoundMode mode) -{ - return float2uint(a, mode); -} - -// intrinsic for division -float fdividef(float a, float b) -{ - return (a / b); -} - -float __internal_accurate_fdividef(float a, float b) -{ - return fdividef(a, b); -} + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; +} + +// float to integer conversion +int float2int(float a, enum cudaRoundMode mode) { + int tmp; + switch (mode) { + case cudaRoundZero: + tmp = truncf(a); + break; + case cudaRoundNearest: + tmp = nearbyintf(a); + break; + case cudaRoundMinInf: + tmp = floorf(a); + break; + case cudaRoundPosInf: + tmp = ceilf(a); + break; + default: + abort(); + } + return tmp; +} + +int __internal_float2int(float a, enum cudaRoundMode mode) { + return float2int(a, mode); +} + +// float to unsigned integer conversion +unsigned int float2uint(float a, enum cudaRoundMode mode) { + unsigned int tmp; + switch (mode) { + case cudaRoundZero: + tmp = truncf(a); + break; + case cudaRoundNearest: + tmp = nearbyintf(a); + break; + case cudaRoundMinInf: + tmp = floorf(a); + break; + case cudaRoundPosInf: + tmp = ceilf(a); + break; + default: + abort(); + } + return tmp; +} + +unsigned int __internal_float2uint(float a, enum cudaRoundMode mode) { + return float2uint(a, mode); +} + +// intrinsic for division +float fdividef(float a, float b) { return (a / b); } + +float __internal_accurate_fdividef(float a, float b) { return fdividef(a, b); } // intrinsic for saturate (clamp values beyond 0 and 1) -float __saturatef(float a) -{ - float b; - if (std::isnan(a)) b = 0.0f; - else if (a >= 1.0f) b = 1.0f; - else if (a <= 0.0f) b = 0.0f; - else b = a; - return b; -} - -// intrinsic for power -float __powf(float a, float b) -{ - return powf(a, b); -} +float __saturatef(float a) { + float b; + if (std::isnan(a)) + b = 0.0f; + else if (a >= 1.0f) + b = 1.0f; + else if (a <= 0.0f) + b = 0.0f; + else + b = a; + return b; +} + +// intrinsic for power +float __powf(float a, float b) { return powf(a, b); } // math functions missing in Mac OSX GCC #ifdef __APPLE__ -int __signbitd(double d) -{ - unsigned long long int u = *((unsigned long long int*)&d); - return ((u & 0x8000000000000000ULL) != 0); +int __signbitd(double d) { + unsigned long long int u = *((unsigned long long int*)&d); + return ((u & 0x8000000000000000ULL) != 0); } -#endif +#endif #undef __CUDACC__ #define __CUDA_INTERNAL_COMPILATION__ @@ -354,15 +357,11 @@ int __signbitd(double d) #undef __attribute__ #endif - } // math functions missing in Mac OSX GCC #ifdef __APPLE__ -int isnanf(float a) -{ - return (std::isnan(a)); -} -#endif +int isnanf(float a) { return (std::isnan(a)); } +#endif #endif diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 7a130ea..ac8af05 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1,5 +1,5 @@ // Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda, Wilson W.L. Fung, -// George L. Yuan, Jimmy Kwa +// George L. Yuan, Jimmy Kwa // The University of British Columbia // All rights reserved. // @@ -8,14 +8,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -31,260 +33,318 @@ #include "instructions.h" #include "ptx_ir.h" class ptx_recognizer; -typedef void * yyscan_t; -#include "ptx.tab.h" -#include "ptx_sim.h" +typedef void *yyscan_t; #include -#include -#include "opcodes.h" -#include "../statwrapper.h" -#include #include +#include +#include +#include "../../libcuda/gpgpu_context.h" #include "../abstract_hardware_model.h" +#include "../gpgpu-sim/gpu-sim.h" +#include "../gpgpusim_entrypoint.h" +#include "../statwrapper.h" +#include "../stream_manager.h" +#include "cuda_device_runtime.h" +#include "decuda_pred_table/decuda_pred_table.h" #include "memory.h" +#include "opcodes.h" #include "ptx-stats.h" +#include "ptx.tab.h" #include "ptx_loader.h" #include "ptx_parser.h" -#include "../gpgpu-sim/gpu-sim.h" #include "ptx_sim.h" -#include "../gpgpusim_entrypoint.h" -#include "decuda_pred_table/decuda_pred_table.h" -#include "../stream_manager.h" -#include "cuda_device_runtime.h" -#include "../../libcuda/gpgpu_context.h" +#include "ptx_sim.h" int g_debug_execution = 0; // Output debug information to file options - -void cuda_sim::ptx_opcocde_latency_options (option_parser_t opp) { - option_parser_register(opp, "-ptx_opcode_latency_int", OPT_CSTR, &opcode_latency_int, - "Opcode latencies for integers " - "Default 1,1,19,25,145", - "1,1,19,25,145"); - option_parser_register(opp, "-ptx_opcode_latency_fp", OPT_CSTR, &opcode_latency_fp, - "Opcode latencies for single precision floating points " - "Default 1,1,1,1,30", - "1,1,1,1,30"); - option_parser_register(opp, "-ptx_opcode_latency_dp", OPT_CSTR, &opcode_latency_dp, - "Opcode latencies for double precision floating points " - "Default 8,8,8,8,335", - "8,8,8,8,335"); - option_parser_register(opp, "-ptx_opcode_latency_sfu", OPT_CSTR, &opcode_latency_sfu, - "Opcode latencies for SFU instructions" - "Default 8", - "8"); - option_parser_register(opp, "-ptx_opcode_latency_tesnor", OPT_CSTR, &opcode_latency_tensor, - "Opcode latencies for Tensor instructions" - "Default 64", - "64"); - option_parser_register(opp, "-ptx_opcode_initiation_int", OPT_CSTR, &opcode_initiation_int, - "Opcode initiation intervals for integers " - "Default 1,1,4,4,32", - "1,1,4,4,32"); - option_parser_register(opp, "-ptx_opcode_initiation_fp", OPT_CSTR, &opcode_initiation_fp, - "Opcode initiation intervals for single precision floating points " - "Default 1,1,1,1,5", - "1,1,1,1,5"); - option_parser_register(opp, "-ptx_opcode_initiation_dp", OPT_CSTR, &opcode_initiation_dp, - "Opcode initiation intervals for double precision floating points " - "Default 8,8,8,8,130", - "8,8,8,8,130"); - option_parser_register(opp, "-ptx_opcode_initiation_sfu", OPT_CSTR, &opcode_initiation_sfu, - "Opcode initiation intervals for sfu instructions" - "Default 8", - "8"); - option_parser_register(opp, "-ptx_opcode_initiation_tensor", OPT_CSTR, &opcode_initiation_tensor, - "Opcode initiation intervals for tensor instructions" - "Default 64", - "64"); - option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str, - "CDP API latency " + "Default 1,1,19,25,145", + "1,1,19,25,145"); + option_parser_register(opp, "-ptx_opcode_latency_fp", OPT_CSTR, + &opcode_latency_fp, + "Opcode latencies for single precision floating " + "points " + "Default 1,1,1,1,30", + "1,1,1,1,30"); + option_parser_register(opp, "-ptx_opcode_latency_dp", OPT_CSTR, + &opcode_latency_dp, + "Opcode latencies for double precision floating " + "points " + "Default 8,8,8,8,335", + "8,8,8,8,335"); + option_parser_register(opp, "-ptx_opcode_latency_sfu", OPT_CSTR, + &opcode_latency_sfu, + "Opcode latencies for SFU instructions" + "Default 8", + "8"); + option_parser_register(opp, "-ptx_opcode_latency_tesnor", OPT_CSTR, + &opcode_latency_tensor, + "Opcode latencies for Tensor instructions" + "Default 64", + "64"); + option_parser_register( + opp, "-ptx_opcode_initiation_int", OPT_CSTR, &opcode_initiation_int, + "Opcode initiation intervals for integers " + "Default 1,1,4,4,32", + "1,1,4,4,32"); + option_parser_register(opp, "-ptx_opcode_initiation_fp", OPT_CSTR, + &opcode_initiation_fp, + "Opcode initiation intervals for single precision " + "floating points " + "Default 1,1,1,1,5", + "1,1,1,1,5"); + option_parser_register(opp, "-ptx_opcode_initiation_dp", OPT_CSTR, + &opcode_initiation_dp, + "Opcode initiation intervals for double precision " + "floating points " + "Default 8,8,8,8,130", + "8,8,8,8,130"); + option_parser_register(opp, "-ptx_opcode_initiation_sfu", OPT_CSTR, + &opcode_initiation_sfu, + "Opcode initiation intervals for sfu instructions" + "Default 8", + "8"); + option_parser_register(opp, "-ptx_opcode_initiation_tensor", OPT_CSTR, + &opcode_initiation_tensor, + "Opcode initiation intervals for tensor instructions" + "Default 64", + "64"); + option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str, + "CDP API latency " - "Default 7200,8000,100,12000,1600", - "7200,8000,100,12000,1600"); -} - -void gpgpu_t::gpgpu_ptx_sim_bindNameToTexture(const char* name, const struct textureReference* texref, int dim, int readmode, int ext) -{ - std::string texname(name); - if (m_NameToTextureRef.find(texname)==m_NameToTextureRef.end()){ - m_NameToTextureRef[texname] = std::set(); - }else{ - const struct textureReference* tr = *m_NameToTextureRef[texname].begin(); - assert(tr!=NULL); - //asserts that all texrefs in set have same fields - assert(tr->normalized==texref->normalized&& - tr->filterMode==texref->filterMode&& - tr->addressMode[0]==texref->addressMode[0]&& - tr->addressMode[1]==texref->addressMode[1]&& - tr->addressMode[2]==texref->addressMode[2]&& - tr->channelDesc.x==texref->channelDesc.x&& - tr->channelDesc.y==texref->channelDesc.y&& - tr->channelDesc.z==texref->channelDesc.z&& - tr->channelDesc.w==texref->channelDesc.w&& - tr->channelDesc.f==texref->channelDesc.f - ); - } - m_NameToTextureRef[texname].insert(texref); - m_TextureRefToName[texref] = texname; - const textureReferenceAttr *texAttr = new textureReferenceAttr(texref, dim, (enum cudaTextureReadMode)readmode, ext); - m_NameToAttribute[texname] = texAttr; -} - -const char* gpgpu_t::gpgpu_ptx_sim_findNamefromTexture(const struct textureReference* texref) -{ - std::map::const_iterator t=m_TextureRefToName.find(texref); - assert( t != m_TextureRefToName.end() ); - return t->second.c_str(); -} - -unsigned int intLOGB2( unsigned int v ) { - unsigned int shift; - unsigned int r; - - r = 0; - - shift = (( v & 0xFFFF0000) != 0 ) << 4; v >>= shift; r |= shift; - shift = (( v & 0xFF00 ) != 0 ) << 3; v >>= shift; r |= shift; - shift = (( v & 0xF0 ) != 0 ) << 2; v >>= shift; r |= shift; - shift = (( v & 0xC ) != 0 ) << 1; v >>= shift; r |= shift; - shift = (( v & 0x2 ) != 0 ) << 0; v >>= shift; r |= shift; - - return r; -} - -void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* texref, const struct cudaArray* array) -{ - std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); - - std::map::const_iterator t=m_NameToCudaArray.find(texname); - //check that there's nothing there first - if(t != m_NameToCudaArray.end()){ - printf("GPGPU-Sim PTX: Warning: binding to texref associated with %s, which was previously bound.\nImplicitly unbinding texref associated to %s first\n", texname.c_str(), texname.c_str()); - } - m_NameToCudaArray[texname] = array; - unsigned int texel_size_bits = array->desc.w + array->desc.x + array->desc.y + array->desc.z; - unsigned int texel_size = texel_size_bits/8; - unsigned int Tx, Ty; - int r; - - printf("GPGPU-Sim PTX: texel size = %d\n", texel_size); - printf("GPGPU-Sim PTX: texture cache linesize = %d\n", m_function_model_config.get_texcache_linesize()); - //first determine base Tx size for given linesize - switch (m_function_model_config.get_texcache_linesize()) { - case 16: Tx = 4; break; - case 32: Tx = 8; break; - case 64: Tx = 8; break; - case 128: Tx = 16; break; - case 256: Tx = 16; break; - default: - printf("GPGPU-Sim PTX: Line size of %d bytes currently not supported.\n", m_function_model_config.get_texcache_linesize()); + "Default 7200,8000,100,12000,1600", + "7200,8000,100,12000,1600"); +} + +void gpgpu_t::gpgpu_ptx_sim_bindNameToTexture( + const char *name, const struct textureReference *texref, int dim, + int readmode, int ext) { + std::string texname(name); + if (m_NameToTextureRef.find(texname) == m_NameToTextureRef.end()) { + m_NameToTextureRef[texname] = std::set(); + } else { + const struct textureReference *tr = *m_NameToTextureRef[texname].begin(); + assert(tr != NULL); + // asserts that all texrefs in set have same fields + assert(tr->normalized == texref->normalized && + tr->filterMode == texref->filterMode && + tr->addressMode[0] == texref->addressMode[0] && + tr->addressMode[1] == texref->addressMode[1] && + tr->addressMode[2] == texref->addressMode[2] && + tr->channelDesc.x == texref->channelDesc.x && + tr->channelDesc.y == texref->channelDesc.y && + tr->channelDesc.z == texref->channelDesc.z && + tr->channelDesc.w == texref->channelDesc.w && + tr->channelDesc.f == texref->channelDesc.f); + } + m_NameToTextureRef[texname].insert(texref); + m_TextureRefToName[texref] = texname; + const textureReferenceAttr *texAttr = new textureReferenceAttr( + texref, dim, (enum cudaTextureReadMode)readmode, ext); + m_NameToAttribute[texname] = texAttr; +} + +const char *gpgpu_t::gpgpu_ptx_sim_findNamefromTexture( + const struct textureReference *texref) { + std::map::const_iterator t = + m_TextureRefToName.find(texref); + assert(t != m_TextureRefToName.end()); + return t->second.c_str(); +} + +unsigned int intLOGB2(unsigned int v) { + unsigned int shift; + unsigned int r; + + r = 0; + + shift = ((v & 0xFFFF0000) != 0) << 4; + v >>= shift; + r |= shift; + shift = ((v & 0xFF00) != 0) << 3; + v >>= shift; + r |= shift; + shift = ((v & 0xF0) != 0) << 2; + v >>= shift; + r |= shift; + shift = ((v & 0xC) != 0) << 1; + v >>= shift; + r |= shift; + shift = ((v & 0x2) != 0) << 0; + v >>= shift; + r |= shift; + + return r; +} + +void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray( + const struct textureReference *texref, const struct cudaArray *array) { + std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); + + std::map::const_iterator t = + m_NameToCudaArray.find(texname); + // check that there's nothing there first + if (t != m_NameToCudaArray.end()) { + printf( + "GPGPU-Sim PTX: Warning: binding to texref associated with %s, which " + "was previously bound.\nImplicitly unbinding texref associated to %s " + "first\n", + texname.c_str(), texname.c_str()); + } + m_NameToCudaArray[texname] = array; + unsigned int texel_size_bits = + array->desc.w + array->desc.x + array->desc.y + array->desc.z; + unsigned int texel_size = texel_size_bits / 8; + unsigned int Tx, Ty; + int r; + + printf("GPGPU-Sim PTX: texel size = %d\n", texel_size); + printf("GPGPU-Sim PTX: texture cache linesize = %d\n", + m_function_model_config.get_texcache_linesize()); + // first determine base Tx size for given linesize + switch (m_function_model_config.get_texcache_linesize()) { + case 16: + Tx = 4; + break; + case 32: + Tx = 8; + break; + case 64: + Tx = 8; + break; + case 128: + Tx = 16; + break; + case 256: + Tx = 16; + break; + default: + printf( + "GPGPU-Sim PTX: Line size of %d bytes currently not supported.\n", + m_function_model_config.get_texcache_linesize()); assert(0); break; - } - r = texel_size >> 2; - //modify base Tx size to take into account size of each texel in bytes - while (r != 0) { - Tx = Tx >> 1; - r = r >> 2; - } - //by now, got the correct Tx size, calculate correct Ty size - Ty = m_function_model_config.get_texcache_linesize()/(Tx*texel_size); - - printf("GPGPU-Sim PTX: Tx = %d; Ty = %d, Tx_numbits = %d, Ty_numbits = %d\n", Tx, Ty, intLOGB2(Tx), intLOGB2(Ty)); - printf("GPGPU-Sim PTX: Texel size = %d bytes; texel_size_numbits = %d\n", texel_size, intLOGB2(texel_size)); - printf("GPGPU-Sim PTX: Binding texture to array starting at devPtr32 = 0x%x\n", array->devPtr32); - printf("GPGPU-Sim PTX: Texel size = %d bytes\n", texel_size); - struct textureInfo* texInfo = (struct textureInfo*) malloc(sizeof(struct textureInfo)); - texInfo->Tx = Tx; - texInfo->Ty = Ty; - texInfo->Tx_numbits = intLOGB2(Tx); - texInfo->Ty_numbits = intLOGB2(Ty); - texInfo->texel_size = texel_size; - texInfo->texel_size_numbits = intLOGB2(texel_size); - m_NameToTextureInfo[texname] = texInfo; -} - -void gpgpu_t::gpgpu_ptx_sim_unbindTexture(const struct textureReference* texref) -{ - //assumes bind-use-unbind-bind-use-unbind pattern - std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); - m_NameToCudaArray.erase(texname); - m_NameToTextureInfo.erase(texname); + } + r = texel_size >> 2; + // modify base Tx size to take into account size of each texel in bytes + while (r != 0) { + Tx = Tx >> 1; + r = r >> 2; + } + // by now, got the correct Tx size, calculate correct Ty size + Ty = m_function_model_config.get_texcache_linesize() / (Tx * texel_size); + + printf( + "GPGPU-Sim PTX: Tx = %d; Ty = %d, Tx_numbits = %d, Ty_numbits = %d\n", + Tx, Ty, intLOGB2(Tx), intLOGB2(Ty)); + printf("GPGPU-Sim PTX: Texel size = %d bytes; texel_size_numbits = %d\n", + texel_size, intLOGB2(texel_size)); + printf( + "GPGPU-Sim PTX: Binding texture to array starting at devPtr32 = 0x%x\n", + array->devPtr32); + printf("GPGPU-Sim PTX: Texel size = %d bytes\n", texel_size); + struct textureInfo *texInfo = + (struct textureInfo *)malloc(sizeof(struct textureInfo)); + texInfo->Tx = Tx; + texInfo->Ty = Ty; + texInfo->Tx_numbits = intLOGB2(Tx); + texInfo->Ty_numbits = intLOGB2(Ty); + texInfo->texel_size = texel_size; + texInfo->texel_size_numbits = intLOGB2(texel_size); + m_NameToTextureInfo[texname] = texInfo; +} + +void gpgpu_t::gpgpu_ptx_sim_unbindTexture( + const struct textureReference *texref) { + // assumes bind-use-unbind-bind-use-unbind pattern + std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); + m_NameToCudaArray.erase(texname); + m_NameToTextureInfo.erase(texname); } #define MAX_INST_SIZE 8 /*bytes*/ -void function_info::ptx_assemble() -{ - if( m_assembled ) { - return; - } - - // get the instructions into instruction memory... - unsigned num_inst = m_instructions.size(); - m_instr_mem_size = MAX_INST_SIZE*(num_inst+1); - m_instr_mem = new ptx_instruction*[ m_instr_mem_size ]; - - printf("GPGPU-Sim PTX: instruction assembly for function \'%s\'... ", m_name.c_str() ); - fflush(stdout); - std::list::iterator i; - - addr_t PC = gpgpu_ctx->func_sim->g_assemble_code_next_pc; // globally unique address (across functions) - // start function on an aligned address - for( unsigned i=0; i < (PC%MAX_INST_SIZE); i++ ) - gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction*)NULL); - PC += PC%MAX_INST_SIZE; - m_start_PC = PC; - - addr_t n=0; // offset in m_instr_mem - //Why s_g_pc_to_insn.size() is needed to reserve additional memory for insts? reserve is cumulative. - //s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + MAX_INST_SIZE*m_instructions.size()); - gpgpu_ctx->s_g_pc_to_insn.reserve(MAX_INST_SIZE*m_instructions.size()); - for ( i=m_instructions.begin(); i != m_instructions.end(); i++ ) { - ptx_instruction *pI = *i; - if ( pI->is_label() ) { - const symbol *l = pI->get_label(); - labels[l->name()] = n; - } else { - gpgpu_ctx->func_sim->g_pc_to_finfo[PC] = this; - m_instr_mem[n] = pI; - gpgpu_ctx->s_g_pc_to_insn.push_back(pI); - assert(pI == gpgpu_ctx->s_g_pc_to_insn[PC]); - pI->set_m_instr_mem_index(n); - pI->set_PC(PC); - assert( pI->inst_size() <= MAX_INST_SIZE ); - for( unsigned i=1; i < pI->inst_size(); i++ ) { - gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction*)NULL); - m_instr_mem[n+i]=NULL; - } - n += pI->inst_size(); - PC += pI->inst_size(); +void function_info::ptx_assemble() { + if (m_assembled) { + return; + } + + // get the instructions into instruction memory... + unsigned num_inst = m_instructions.size(); + m_instr_mem_size = MAX_INST_SIZE * (num_inst + 1); + m_instr_mem = new ptx_instruction *[m_instr_mem_size]; + + printf("GPGPU-Sim PTX: instruction assembly for function \'%s\'... ", + m_name.c_str()); + fflush(stdout); + std::list::iterator i; + + addr_t PC = gpgpu_ctx->func_sim->g_assemble_code_next_pc; // globally unique + // address (across + // functions) + // start function on an aligned address + for (unsigned i = 0; i < (PC % MAX_INST_SIZE); i++) + gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction *)NULL); + PC += PC % MAX_INST_SIZE; + m_start_PC = PC; + + addr_t n = 0; // offset in m_instr_mem + // Why s_g_pc_to_insn.size() is needed to reserve additional memory for insts? + // reserve is cumulative. + // s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + + // MAX_INST_SIZE*m_instructions.size()); + gpgpu_ctx->s_g_pc_to_insn.reserve(MAX_INST_SIZE * m_instructions.size()); + for (i = m_instructions.begin(); i != m_instructions.end(); i++) { + ptx_instruction *pI = *i; + if (pI->is_label()) { + const symbol *l = pI->get_label(); + labels[l->name()] = n; + } else { + gpgpu_ctx->func_sim->g_pc_to_finfo[PC] = this; + m_instr_mem[n] = pI; + gpgpu_ctx->s_g_pc_to_insn.push_back(pI); + assert(pI == gpgpu_ctx->s_g_pc_to_insn[PC]); + pI->set_m_instr_mem_index(n); + pI->set_PC(PC); + assert(pI->inst_size() <= MAX_INST_SIZE); + for (unsigned i = 1; i < pI->inst_size(); i++) { + gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction *)NULL); + m_instr_mem[n + i] = NULL; } - } - gpgpu_ctx->func_sim->g_assemble_code_next_pc=PC; - for ( unsigned ii=0; ii < n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions - ptx_instruction *pI = m_instr_mem[ii]; - if ( pI->get_opcode() == BRA_OP || pI->get_opcode() == BREAKADDR_OP || pI->get_opcode() == CALLP_OP) { - operand_info &target = pI->dst(); //get operand, e.g. target name - if ( labels.find(target.name()) == labels.end() ) { - printf("GPGPU-Sim PTX: Loader error (%s:%u): Branch label \"%s\" does not appear in assembly code.", - pI->source_file(),pI->source_line(), target.name().c_str() ); - abort(); - } - unsigned index = labels[ target.name() ]; //determine address from name - unsigned PC = m_instr_mem[index]->get_PC(); - m_symtab->set_label_address( target.get_symbol(), PC ); - target.set_type(label_t); + n += pI->inst_size(); + PC += pI->inst_size(); + } + } + gpgpu_ctx->func_sim->g_assemble_code_next_pc = PC; + for (unsigned ii = 0; ii < n; + ii += m_instr_mem[ii]->inst_size()) { // handle branch instructions + ptx_instruction *pI = m_instr_mem[ii]; + if (pI->get_opcode() == BRA_OP || pI->get_opcode() == BREAKADDR_OP || + pI->get_opcode() == CALLP_OP) { + operand_info &target = pI->dst(); // get operand, e.g. target name + if (labels.find(target.name()) == labels.end()) { + printf( + "GPGPU-Sim PTX: Loader error (%s:%u): Branch label \"%s\" does not " + "appear in assembly code.", + pI->source_file(), pI->source_line(), target.name().c_str()); + abort(); } - } - m_n = n; - printf(" done.\n"); - fflush(stdout); + unsigned index = labels[target.name()]; // determine address from name + unsigned PC = m_instr_mem[index]->get_PC(); + m_symtab->set_label_address(target.get_symbol(), PC); + target.set_type(label_t); + } + } + m_n = n; + printf(" done.\n"); + fflush(stdout); - //disable pdom analysis here and do it at runtime +// disable pdom analysis here and do it at runtime #if 0 printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", m_name.c_str() ); create_basic_blocks(); @@ -323,2247 +383,2448 @@ void function_info::ptx_assemble() #endif } -addr_t shared_to_generic( unsigned smid, addr_t addr ) -{ - assert( addr < SHARED_MEM_SIZE_MAX ); - return SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX + addr; -} - -addr_t global_to_generic( addr_t addr ) -{ - return addr; -} - -bool isspace_shared( unsigned smid, addr_t addr ) -{ - addr_t start = SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX; - addr_t end = SHARED_GENERIC_START + (smid+1)*SHARED_MEM_SIZE_MAX; - if( (addr >= end) || (addr < start) ) - return false; - return true; -} - -bool isspace_global( addr_t addr ) -{ - return (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT); -} - -memory_space_t whichspace( addr_t addr ) -{ - if( (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT) ) { - return global_space; - } else if( addr >= SHARED_GENERIC_START ) { - return shared_space; - } else { - return local_space; - } -} - -addr_t generic_to_shared( unsigned smid, addr_t addr ) -{ - assert(isspace_shared(smid,addr)); - return addr - (SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX); -} - -addr_t local_to_generic( unsigned smid, unsigned hwtid, addr_t addr ) -{ - assert(addr < LOCAL_MEM_SIZE_MAX); - return LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid) + addr; -} - -bool isspace_local( unsigned smid, unsigned hwtid, addr_t addr ) -{ - addr_t start = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid); - addr_t end = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * (hwtid+1)); - if( (addr >= end) || (addr < start) ) - return false; - return true; -} - -addr_t generic_to_local( unsigned smid, unsigned hwtid, addr_t addr ) -{ - assert(isspace_local(smid,hwtid,addr)); - return addr - (LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid)); -} - -addr_t generic_to_global( addr_t addr ) -{ - return addr; -} - - -void* gpgpu_t::gpu_malloc( size_t size ) -{ - unsigned long long result = m_dev_malloc; - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address 0x%Lx\n", size, m_dev_malloc ); - fflush(stdout); - } - m_dev_malloc += size; - if (size%256) m_dev_malloc += (256 - size%256); //align to 256 byte boundaries - return(void*) result; -} - -void* gpgpu_t::gpu_mallocarray( size_t size ) -{ - unsigned long long result = m_dev_malloc; - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address 0x%Lx\n", size, m_dev_malloc ); - fflush(stdout); - } - m_dev_malloc += size; - if (size%256) m_dev_malloc += (256 - size%256); //align to 256 byte boundaries - return(void*) result; -} - - -void gpgpu_t::memcpy_to_gpu( size_t dst_start_addr, const void *src, size_t count ) -{ - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: copying %zu bytes from CPU[0x%Lx] to GPU[0x%Lx] ... ", count, (unsigned long long) src, (unsigned long long) dst_start_addr ); - fflush(stdout); - } - char *src_data = (char*)src; - for (unsigned n=0; n < count; n ++ ) - m_global_mem->write(dst_start_addr+n,1, src_data+n,NULL,NULL); - - // Copy into the performance model. - //extern gpgpu_sim* g_the_gpu; - gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(dst_start_addr, count); - if(g_debug_execution >= 3) { - printf( " done.\n"); - fflush(stdout); - } -} - -void gpgpu_t::memcpy_from_gpu( void *dst, size_t src_start_addr, size_t count ) -{ - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to CPU[0x%Lx] ...", count, (unsigned long long) src_start_addr, (unsigned long long) dst ); - fflush(stdout); - } - unsigned char *dst_data = (unsigned char*)dst; - for (unsigned n=0; n < count; n ++ ) - m_global_mem->read(src_start_addr+n,1,dst_data+n); - - // Copy into the performance model. - //extern gpgpu_sim* g_the_gpu; - gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(src_start_addr, count); - if(g_debug_execution >= 3) { - printf( " done.\n"); - fflush(stdout); - } -} - -void gpgpu_t::memcpy_gpu_to_gpu( size_t dst, size_t src, size_t count ) -{ - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to GPU[0x%Lx] ...", count, - (unsigned long long) src, (unsigned long long) dst ); - fflush(stdout); - } - for (unsigned n=0; n < count; n ++ ) { - unsigned char tmp; - m_global_mem->read(src+n,1,&tmp); - m_global_mem->write(dst+n,1, &tmp,NULL,NULL); - } - if(g_debug_execution >= 3) { - printf( " done.\n"); - fflush(stdout); - } -} - -void gpgpu_t::gpu_memset( size_t dst_start_addr, int c, size_t count ) -{ - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: setting %zu bytes of memory to 0x%x starting at 0x%Lx... ", - count, (unsigned char) c, (unsigned long long) dst_start_addr ); - fflush(stdout); - } - unsigned char c_value = (unsigned char)c; - for (unsigned n=0; n < count; n ++ ) - m_global_mem->write(dst_start_addr+n,1,&c_value,NULL,NULL); - if(g_debug_execution >= 3) { - printf( " done.\n"); - fflush(stdout); - } -} +addr_t shared_to_generic(unsigned smid, addr_t addr) { + assert(addr < SHARED_MEM_SIZE_MAX); + return SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX + addr; +} + +addr_t global_to_generic(addr_t addr) { return addr; } -void cuda_sim::ptx_print_insn( address_type pc, FILE *fp ) -{ - std::map::iterator f = g_pc_to_finfo.find(pc); - if( f == g_pc_to_finfo.end() ) { - fprintf(fp,"", pc ); - return; - } - function_info *finfo = f->second; - assert( finfo ); - finfo->print_insn(pc,fp); -} - -std::string cuda_sim::ptx_get_insn_str( address_type pc ) -{ - std::map::iterator f = g_pc_to_finfo.find(pc); - if( f == g_pc_to_finfo.end() ) { - #define STR_SIZE 255 - char buff[STR_SIZE]; - buff[STR_SIZE - 1] = '\0'; - snprintf(buff, STR_SIZE,"", pc ); - return std::string(buff); - } - function_info *finfo = f->second; - assert( finfo ); - return finfo->get_insn_str(pc); -} - -void ptx_instruction::set_fp_or_int_archop(){ - oprnd_type=UN_OP; - if((m_opcode == MEMBAR_OP)||(m_opcode == SSY_OP )||(m_opcode == BRA_OP) || (m_opcode == BAR_OP) || (m_opcode == RET_OP) || (m_opcode == RETP_OP) || (m_opcode == NOP_OP) || (m_opcode == EXIT_OP) || (m_opcode == CALLP_OP) || (m_opcode == CALL_OP)){ - // do nothing - }else if((m_opcode == CVT_OP || m_opcode == SET_OP || m_opcode == SLCT_OP)){ - if(get_type2()==F16_TYPE || get_type2()==F32_TYPE || get_type2() == F64_TYPE || get_type2() == FF64_TYPE){ - oprnd_type= FP_OP; - }else oprnd_type=INT_OP; - - }else{ - if(get_type()==F16_TYPE || get_type()==F32_TYPE || get_type() == F64_TYPE || get_type() == FF64_TYPE){ - oprnd_type= FP_OP; - }else oprnd_type=INT_OP; - } -} -void ptx_instruction::set_mul_div_or_other_archop(){ - sp_op=OTHER_OP; - if((m_opcode != MEMBAR_OP) && (m_opcode != SSY_OP) && (m_opcode != BRA_OP) && (m_opcode != BAR_OP) && (m_opcode != EXIT_OP) && (m_opcode != NOP_OP) && (m_opcode != RETP_OP) && (m_opcode != RET_OP) && (m_opcode != CALLP_OP) && (m_opcode != CALL_OP)){ - if(get_type()==F32_TYPE || get_type() == F64_TYPE || get_type() == FF64_TYPE){ - switch(get_opcode()){ - case MUL_OP: - case MAD_OP: - sp_op=FP_MUL_OP; - break; - case DIV_OP: - sp_op=FP_DIV_OP; - break; - case LG2_OP: - sp_op=FP_LG_OP; - break; - case RSQRT_OP: - case SQRT_OP: - sp_op=FP_SQRT_OP; - break; - case RCP_OP: - sp_op=FP_DIV_OP; - break; - case SIN_OP: - case COS_OP: - sp_op=FP_SIN_OP; - break; - case EX2_OP: - sp_op=FP_EXP_OP; - break; - default: - if((op==ALU_OP)||(op==TENSOR_CORE_OP)) - sp_op=FP__OP; - break; - - } - }else { - switch(get_opcode()){ - case MUL24_OP: - case MAD24_OP: - sp_op=INT_MUL24_OP; - break; - case MUL_OP: - case MAD_OP: - if(get_type()==U32_TYPE || get_type()==S32_TYPE || get_type()==B32_TYPE) - sp_op=INT_MUL32_OP; - else - sp_op=INT_MUL_OP; - break; - case DIV_OP: - sp_op=INT_DIV_OP; - break; - default: - if((op==ALU_OP)) - sp_op=INT__OP; - break; - } - } - } - -} - - - -void ptx_instruction::set_bar_type() -{ - if(m_opcode==BAR_OP) { - switch(m_barrier_op){ - case SYNC_OPTION: - bar_type = SYNC; - break; - case ARRIVE_OPTION: - bar_type = ARRIVE; - break; - case RED_OPTION: - bar_type = RED; - switch(m_atomic_spec){ - case ATOMIC_POPC: - red_type = POPC_RED; - break; - case ATOMIC_AND: - red_type = AND_RED; - break; - case ATOMIC_OR: - red_type = OR_RED; - break; - } - break; - default: - abort(); - } - } - else if(m_opcode==SST_OP) { - bar_type = SYNC; - } -} - - -void ptx_instruction::set_opcode_and_latency() -{ - unsigned int_latency[5]; - unsigned fp_latency[5]; - unsigned dp_latency[5]; - unsigned sfu_latency; - unsigned tensor_latency; - unsigned int_init[5]; - unsigned fp_init[5]; - unsigned dp_init[5]; - unsigned sfu_init; - unsigned tensor_init; - /* - * [0] ADD,SUB - * [1] MAX,Min - * [2] MUL - * [3] MAD - * [4] DIV - */ - sscanf(gpgpu_ctx->func_sim->opcode_latency_int, "%u,%u,%u,%u,%u", - &int_latency[0],&int_latency[1],&int_latency[2], - &int_latency[3],&int_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_fp, "%u,%u,%u,%u,%u", - &fp_latency[0],&fp_latency[1],&fp_latency[2], - &fp_latency[3],&fp_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_dp, "%u,%u,%u,%u,%u", - &dp_latency[0],&dp_latency[1],&dp_latency[2], - &dp_latency[3],&dp_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_sfu, "%u", - &sfu_latency); - sscanf(gpgpu_ctx->func_sim->opcode_latency_tensor, "%u", - &tensor_latency); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_int, "%u,%u,%u,%u,%u", - &int_init[0],&int_init[1],&int_init[2], - &int_init[3],&int_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_fp, "%u,%u,%u,%u,%u", - &fp_init[0],&fp_init[1],&fp_init[2], - &fp_init[3],&fp_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_dp, "%u,%u,%u,%u,%u", - &dp_init[0],&dp_init[1],&dp_init[2], - &dp_init[3],&dp_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_sfu, "%u", - &sfu_init); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_tensor, "%u", - &tensor_init); - sscanf(gpgpu_ctx->func_sim->cdp_latency_str, "%u,%u,%u,%u,%u", - &gpgpu_ctx->func_sim->cdp_latency[0], - &gpgpu_ctx->func_sim->cdp_latency[1], - &gpgpu_ctx->func_sim->cdp_latency[2], - &gpgpu_ctx->func_sim->cdp_latency[3], - &gpgpu_ctx->func_sim->cdp_latency[4]); - - if(!m_operands.empty()){ - std::vector::iterator it; - for(it=++m_operands.begin();it!=m_operands.end();it++){ - num_operands++; - if((it->is_reg() || it->is_vector())){ - num_regs++; - } - } - } - op = ALU_OP; - mem_op= NOT_TEX; - initiation_interval = latency = 1; - switch( m_opcode ) { - case MOV_OP: - assert( !(has_memory_read() && has_memory_write()) ); - if ( has_memory_read() ) op = LOAD_OP; - if ( has_memory_write() ) op = STORE_OP; - break; - case LD_OP: op = LOAD_OP; break; - case MMA_LD_OP: op = TENSOR_CORE_LOAD_OP; break; - case LDU_OP: op = LOAD_OP; break; - case ST_OP: op = STORE_OP; break; - case MMA_ST_OP: op = TENSOR_CORE_STORE_OP; break; - case BRA_OP: op = BRANCH_OP; break; - case BREAKADDR_OP: op = BRANCH_OP; break; - case TEX_OP: op = LOAD_OP; mem_op=TEX; break; - case ATOM_OP: op = LOAD_OP; break; - case BAR_OP: op = BARRIER_OP; break; - case SST_OP: op = BARRIER_OP; break; - case MEMBAR_OP: op = MEMORY_BARRIER_OP; break; - case CALL_OP: - { - if(m_is_printf || m_is_cdp) { - op = ALU_OP; - } - else - op = CALL_OPS; - break; - } - case CALLP_OP: - { - if(m_is_printf || m_is_cdp) { - op = ALU_OP; - } - else - op = CALL_OPS; - break; - } - case RET_OP: case RETP_OP: op = RET_OPS;break; - case ADD_OP: case ADDP_OP: case ADDC_OP: case SUB_OP: case SUBC_OP: - //ADD,SUB latency - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[0]; - initiation_interval = fp_init[0]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[0]; - initiation_interval = dp_init[0]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[0]; - initiation_interval = int_init[0]; - op = INTP_OP; - break; - } - break; - case MAX_OP: case MIN_OP: - //MAX,MIN latency - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[1]; - initiation_interval = fp_init[1]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[1]; - initiation_interval = dp_init[1]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[1]; - initiation_interval = int_init[1]; - op = INTP_OP; - break; - } - break; - case MUL_OP: - //MUL latency - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[2]; - initiation_interval = fp_init[2]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[2]; - initiation_interval = dp_init[2]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[2]; - initiation_interval = int_init[2]; - op = INTP_OP; - break; - } - break; - case MAD_OP: case MADC_OP: case MADP_OP: - //MAD latency - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[3]; - initiation_interval = fp_init[3]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[3]; - initiation_interval = dp_init[3]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[3]; - initiation_interval = int_init[3]; - op = INTP_OP; - break; - } - break; - case DIV_OP: - // Floating point only - op = SFU_OP; - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[4]; - initiation_interval = fp_init[4]; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[4]; - initiation_interval = dp_init[4]; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[4]; - initiation_interval = int_init[4]; - break; - } - break; - case SQRT_OP: case SIN_OP: case COS_OP: case EX2_OP: case LG2_OP: case RSQRT_OP: case RCP_OP: - latency = sfu_latency; - initiation_interval = sfu_init; +bool isspace_shared(unsigned smid, addr_t addr) { + addr_t start = SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX; + addr_t end = SHARED_GENERIC_START + (smid + 1) * SHARED_MEM_SIZE_MAX; + if ((addr >= end) || (addr < start)) return false; + return true; +} + +bool isspace_global(addr_t addr) { + return (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT); +} + +memory_space_t whichspace(addr_t addr) { + if ((addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT)) { + return global_space; + } else if (addr >= SHARED_GENERIC_START) { + return shared_space; + } else { + return local_space; + } +} + +addr_t generic_to_shared(unsigned smid, addr_t addr) { + assert(isspace_shared(smid, addr)); + return addr - (SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX); +} + +addr_t local_to_generic(unsigned smid, unsigned hwtid, addr_t addr) { + assert(addr < LOCAL_MEM_SIZE_MAX); + return LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + + (LOCAL_MEM_SIZE_MAX * hwtid) + addr; +} + +bool isspace_local(unsigned smid, unsigned hwtid, addr_t addr) { + addr_t start = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + + (LOCAL_MEM_SIZE_MAX * hwtid); + addr_t end = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + + (LOCAL_MEM_SIZE_MAX * (hwtid + 1)); + if ((addr >= end) || (addr < start)) return false; + return true; +} + +addr_t generic_to_local(unsigned smid, unsigned hwtid, addr_t addr) { + assert(isspace_local(smid, hwtid, addr)); + return addr - (LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + + (LOCAL_MEM_SIZE_MAX * hwtid)); +} + +addr_t generic_to_global(addr_t addr) { return addr; } + +void *gpgpu_t::gpu_malloc(size_t size) { + unsigned long long result = m_dev_malloc; + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address " + "0x%Lx\n", + size, m_dev_malloc); + fflush(stdout); + } + m_dev_malloc += size; + if (size % 256) + m_dev_malloc += (256 - size % 256); // align to 256 byte boundaries + return (void *)result; +} + +void *gpgpu_t::gpu_mallocarray(size_t size) { + unsigned long long result = m_dev_malloc; + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address " + "0x%Lx\n", + size, m_dev_malloc); + fflush(stdout); + } + m_dev_malloc += size; + if (size % 256) + m_dev_malloc += (256 - size % 256); // align to 256 byte boundaries + return (void *)result; +} + +void gpgpu_t::memcpy_to_gpu(size_t dst_start_addr, const void *src, + size_t count) { + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim PTX: copying %zu bytes from CPU[0x%Lx] to GPU[0x%Lx] ... ", + count, (unsigned long long)src, (unsigned long long)dst_start_addr); + fflush(stdout); + } + char *src_data = (char *)src; + for (unsigned n = 0; n < count; n++) + m_global_mem->write(dst_start_addr + n, 1, src_data + n, NULL, NULL); + + // Copy into the performance model. + // extern gpgpu_sim* g_the_gpu; + gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(dst_start_addr, count); + if (g_debug_execution >= 3) { + printf(" done.\n"); + fflush(stdout); + } +} + +void gpgpu_t::memcpy_from_gpu(void *dst, size_t src_start_addr, size_t count) { + if (g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to CPU[0x%Lx] ...", + count, (unsigned long long)src_start_addr, (unsigned long long)dst); + fflush(stdout); + } + unsigned char *dst_data = (unsigned char *)dst; + for (unsigned n = 0; n < count; n++) + m_global_mem->read(src_start_addr + n, 1, dst_data + n); + + // Copy into the performance model. + // extern gpgpu_sim* g_the_gpu; + gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(src_start_addr, count); + if (g_debug_execution >= 3) { + printf(" done.\n"); + fflush(stdout); + } +} + +void gpgpu_t::memcpy_gpu_to_gpu(size_t dst, size_t src, size_t count) { + if (g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to GPU[0x%Lx] ...", + count, (unsigned long long)src, (unsigned long long)dst); + fflush(stdout); + } + for (unsigned n = 0; n < count; n++) { + unsigned char tmp; + m_global_mem->read(src + n, 1, &tmp); + m_global_mem->write(dst + n, 1, &tmp, NULL, NULL); + } + if (g_debug_execution >= 3) { + printf(" done.\n"); + fflush(stdout); + } +} + +void gpgpu_t::gpu_memset(size_t dst_start_addr, int c, size_t count) { + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim PTX: setting %zu bytes of memory to 0x%x starting at " + "0x%Lx... ", + count, (unsigned char)c, (unsigned long long)dst_start_addr); + fflush(stdout); + } + unsigned char c_value = (unsigned char)c; + for (unsigned n = 0; n < count; n++) + m_global_mem->write(dst_start_addr + n, 1, &c_value, NULL, NULL); + if (g_debug_execution >= 3) { + printf(" done.\n"); + fflush(stdout); + } +} + +void cuda_sim::ptx_print_insn(address_type pc, FILE *fp) { + std::map::iterator f = g_pc_to_finfo.find(pc); + if (f == g_pc_to_finfo.end()) { + fprintf(fp, "", pc); + return; + } + function_info *finfo = f->second; + assert(finfo); + finfo->print_insn(pc, fp); +} + +std::string cuda_sim::ptx_get_insn_str(address_type pc) { + std::map::iterator f = g_pc_to_finfo.find(pc); + if (f == g_pc_to_finfo.end()) { +#define STR_SIZE 255 + char buff[STR_SIZE]; + buff[STR_SIZE - 1] = '\0'; + snprintf(buff, STR_SIZE, "", pc); + return std::string(buff); + } + function_info *finfo = f->second; + assert(finfo); + return finfo->get_insn_str(pc); +} + +void ptx_instruction::set_fp_or_int_archop() { + oprnd_type = UN_OP; + if ((m_opcode == MEMBAR_OP) || (m_opcode == SSY_OP) || (m_opcode == BRA_OP) || + (m_opcode == BAR_OP) || (m_opcode == RET_OP) || (m_opcode == RETP_OP) || + (m_opcode == NOP_OP) || (m_opcode == EXIT_OP) || (m_opcode == CALLP_OP) || + (m_opcode == CALL_OP)) { + // do nothing + } else if ((m_opcode == CVT_OP || m_opcode == SET_OP || + m_opcode == SLCT_OP)) { + if (get_type2() == F16_TYPE || get_type2() == F32_TYPE || + get_type2() == F64_TYPE || get_type2() == FF64_TYPE) { + oprnd_type = FP_OP; + } else + oprnd_type = INT_OP; + + } else { + if (get_type() == F16_TYPE || get_type() == F32_TYPE || + get_type() == F64_TYPE || get_type() == FF64_TYPE) { + oprnd_type = FP_OP; + } else + oprnd_type = INT_OP; + } +} +void ptx_instruction::set_mul_div_or_other_archop() { + sp_op = OTHER_OP; + if ((m_opcode != MEMBAR_OP) && (m_opcode != SSY_OP) && (m_opcode != BRA_OP) && + (m_opcode != BAR_OP) && (m_opcode != EXIT_OP) && (m_opcode != NOP_OP) && + (m_opcode != RETP_OP) && (m_opcode != RET_OP) && (m_opcode != CALLP_OP) && + (m_opcode != CALL_OP)) { + if (get_type() == F32_TYPE || get_type() == F64_TYPE || + get_type() == FF64_TYPE) { + switch (get_opcode()) { + case MUL_OP: + case MAD_OP: + sp_op = FP_MUL_OP; + break; + case DIV_OP: + sp_op = FP_DIV_OP; + break; + case LG2_OP: + sp_op = FP_LG_OP; + break; + case RSQRT_OP: + case SQRT_OP: + sp_op = FP_SQRT_OP; + break; + case RCP_OP: + sp_op = FP_DIV_OP; + break; + case SIN_OP: + case COS_OP: + sp_op = FP_SIN_OP; + break; + case EX2_OP: + sp_op = FP_EXP_OP; + break; + default: + if ((op == ALU_OP) || (op == TENSOR_CORE_OP)) sp_op = FP__OP; + break; + } + } else { + switch (get_opcode()) { + case MUL24_OP: + case MAD24_OP: + sp_op = INT_MUL24_OP; + break; + case MUL_OP: + case MAD_OP: + if (get_type() == U32_TYPE || get_type() == S32_TYPE || + get_type() == B32_TYPE) + sp_op = INT_MUL32_OP; + else + sp_op = INT_MUL_OP; + break; + case DIV_OP: + sp_op = INT_DIV_OP; + break; + default: + if ((op == ALU_OP)) sp_op = INT__OP; + break; + } + } + } +} + +void ptx_instruction::set_bar_type() { + if (m_opcode == BAR_OP) { + switch (m_barrier_op) { + case SYNC_OPTION: + bar_type = SYNC; + break; + case ARRIVE_OPTION: + bar_type = ARRIVE; + break; + case RED_OPTION: + bar_type = RED; + switch (m_atomic_spec) { + case ATOMIC_POPC: + red_type = POPC_RED; + break; + case ATOMIC_AND: + red_type = AND_RED; + break; + case ATOMIC_OR: + red_type = OR_RED; + break; + } + break; + default: + abort(); + } + } else if (m_opcode == SST_OP) { + bar_type = SYNC; + } +} + +void ptx_instruction::set_opcode_and_latency() { + unsigned int_latency[5]; + unsigned fp_latency[5]; + unsigned dp_latency[5]; + unsigned sfu_latency; + unsigned tensor_latency; + unsigned int_init[5]; + unsigned fp_init[5]; + unsigned dp_init[5]; + unsigned sfu_init; + unsigned tensor_init; + /* + * [0] ADD,SUB + * [1] MAX,Min + * [2] MUL + * [3] MAD + * [4] DIV + */ + sscanf(gpgpu_ctx->func_sim->opcode_latency_int, "%u,%u,%u,%u,%u", + &int_latency[0], &int_latency[1], &int_latency[2], &int_latency[3], + &int_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_fp, "%u,%u,%u,%u,%u", + &fp_latency[0], &fp_latency[1], &fp_latency[2], &fp_latency[3], + &fp_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_dp, "%u,%u,%u,%u,%u", + &dp_latency[0], &dp_latency[1], &dp_latency[2], &dp_latency[3], + &dp_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_sfu, "%u", &sfu_latency); + sscanf(gpgpu_ctx->func_sim->opcode_latency_tensor, "%u", &tensor_latency); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_int, "%u,%u,%u,%u,%u", + &int_init[0], &int_init[1], &int_init[2], &int_init[3], &int_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_fp, "%u,%u,%u,%u,%u", + &fp_init[0], &fp_init[1], &fp_init[2], &fp_init[3], &fp_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_dp, "%u,%u,%u,%u,%u", + &dp_init[0], &dp_init[1], &dp_init[2], &dp_init[3], &dp_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_sfu, "%u", &sfu_init); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_tensor, "%u", &tensor_init); + sscanf(gpgpu_ctx->func_sim->cdp_latency_str, "%u,%u,%u,%u,%u", + &gpgpu_ctx->func_sim->cdp_latency[0], + &gpgpu_ctx->func_sim->cdp_latency[1], + &gpgpu_ctx->func_sim->cdp_latency[2], + &gpgpu_ctx->func_sim->cdp_latency[3], + &gpgpu_ctx->func_sim->cdp_latency[4]); + + if (!m_operands.empty()) { + std::vector::iterator it; + for (it = ++m_operands.begin(); it != m_operands.end(); it++) { + num_operands++; + if ((it->is_reg() || it->is_vector())) { + num_regs++; + } + } + } + op = ALU_OP; + mem_op = NOT_TEX; + initiation_interval = latency = 1; + switch (m_opcode) { + case MOV_OP: + assert(!(has_memory_read() && has_memory_write())); + if (has_memory_read()) op = LOAD_OP; + if (has_memory_write()) op = STORE_OP; + break; + case LD_OP: + op = LOAD_OP; + break; + case MMA_LD_OP: + op = TENSOR_CORE_LOAD_OP; + break; + case LDU_OP: + op = LOAD_OP; + break; + case ST_OP: + op = STORE_OP; + break; + case MMA_ST_OP: + op = TENSOR_CORE_STORE_OP; + break; + case BRA_OP: + op = BRANCH_OP; + break; + case BREAKADDR_OP: + op = BRANCH_OP; + break; + case TEX_OP: + op = LOAD_OP; + mem_op = TEX; + break; + case ATOM_OP: + op = LOAD_OP; + break; + case BAR_OP: + op = BARRIER_OP; + break; + case SST_OP: + op = BARRIER_OP; + break; + case MEMBAR_OP: + op = MEMORY_BARRIER_OP; + break; + case CALL_OP: { + if (m_is_printf || m_is_cdp) { + op = ALU_OP; + } else + op = CALL_OPS; + break; + } + case CALLP_OP: { + if (m_is_printf || m_is_cdp) { + op = ALU_OP; + } else + op = CALL_OPS; + break; + } + case RET_OP: + case RETP_OP: + op = RET_OPS; + break; + case ADD_OP: + case ADDP_OP: + case ADDC_OP: + case SUB_OP: + case SUBC_OP: + // ADD,SUB latency + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[0]; + initiation_interval = fp_init[0]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[0]; + initiation_interval = dp_init[0]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[0]; + initiation_interval = int_init[0]; + op = INTP_OP; + break; + } + break; + case MAX_OP: + case MIN_OP: + // MAX,MIN latency + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[1]; + initiation_interval = fp_init[1]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[1]; + initiation_interval = dp_init[1]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[1]; + initiation_interval = int_init[1]; + op = INTP_OP; + break; + } + break; + case MUL_OP: + // MUL latency + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[2]; + initiation_interval = fp_init[2]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[2]; + initiation_interval = dp_init[2]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[2]; + initiation_interval = int_init[2]; + op = INTP_OP; + break; + } + break; + case MAD_OP: + case MADC_OP: + case MADP_OP: + // MAD latency + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[3]; + initiation_interval = fp_init[3]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[3]; + initiation_interval = dp_init[3]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[3]; + initiation_interval = int_init[3]; + op = INTP_OP; + break; + } + break; + case DIV_OP: + // Floating point only op = SFU_OP; + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[4]; + initiation_interval = fp_init[4]; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[4]; + initiation_interval = dp_init[4]; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[4]; + initiation_interval = int_init[4]; + break; + } break; - case MMA_OP: - latency = tensor_latency; - initiation_interval = tensor_init; - op=TENSOR_CORE_OP; - break; - case SHFL_OP: - latency = 4; - initiation_interval = 4; - break; - default: - break; - } - set_fp_or_int_archop(); - set_mul_div_or_other_archop(); - -} - -void ptx_thread_info::ptx_fetch_inst( inst_t &inst ) const -{ - addr_t pc = get_pc(); - const ptx_instruction *pI = m_func_info->get_instruction(pc); - inst = (const inst_t&)*pI; - assert( inst.valid() ); -} - -static unsigned datatype2size( unsigned data_type ) -{ - unsigned data_size; - switch ( data_type ) { - case B8_TYPE: - case S8_TYPE: - case U8_TYPE: - data_size = 1; break; - case B16_TYPE: - case S16_TYPE: - case U16_TYPE: - case F16_TYPE: - data_size = 2; break; - case B32_TYPE: - case S32_TYPE: - case U32_TYPE: - case F32_TYPE: - data_size = 4; break; - case B64_TYPE: - case BB64_TYPE: - case S64_TYPE: - case U64_TYPE: - case F64_TYPE: - case FF64_TYPE: - data_size = 8; break; - case BB128_TYPE: - data_size = 16; break; - default: assert(0); break; - } - return data_size; + case SQRT_OP: + case SIN_OP: + case COS_OP: + case EX2_OP: + case LG2_OP: + case RSQRT_OP: + case RCP_OP: + latency = sfu_latency; + initiation_interval = sfu_init; + op = SFU_OP; + break; + case MMA_OP: + latency = tensor_latency; + initiation_interval = tensor_init; + op = TENSOR_CORE_OP; + break; + case SHFL_OP: + latency = 4; + initiation_interval = 4; + break; + default: + break; + } + set_fp_or_int_archop(); + set_mul_div_or_other_archop(); } -void ptx_instruction::pre_decode() -{ - pc = m_PC; - isize = m_inst_size; - for(unsigned i=0; iget_instruction(pc); + inst = (const inst_t &)*pI; + assert(inst.valid()); +} - switch ( get_opcode() ) { -#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break; -#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break; +static unsigned datatype2size(unsigned data_type) { + unsigned data_size; + switch (data_type) { + case B8_TYPE: + case S8_TYPE: + case U8_TYPE: + data_size = 1; + break; + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: + case F16_TYPE: + data_size = 2; + break; + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: + case F32_TYPE: + data_size = 4; + break; + case B64_TYPE: + case BB64_TYPE: + case S64_TYPE: + case U64_TYPE: + case F64_TYPE: + case FF64_TYPE: + data_size = 8; + break; + case BB128_TYPE: + data_size = 16; + break; + default: + assert(0); + break; + } + return data_size; +} + +void ptx_instruction::pre_decode() { + pc = m_PC; + isize = m_inst_size; + for (unsigned i = 0; i < MAX_OUTPUT_VALUES; i++) { + out[i] = 0; + } + for (unsigned i = 0; i < MAX_INPUT_VALUES; i++) { + in[i] = 0; + } + incount = 0; + outcount = 0; + is_vectorin = 0; + is_vectorout = 0; + std::fill_n(arch_reg.src, MAX_REG_OPERANDS, -1); + std::fill_n(arch_reg.dst, MAX_REG_OPERANDS, -1); + pred = 0; + ar1 = 0; + ar2 = 0; + space = m_space_spec; + memory_op = no_memory_op; + data_size = 0; + if (has_memory_read() || has_memory_write()) { + unsigned to_type = get_type(); + data_size = datatype2size(to_type); + memory_op = has_memory_read() ? memory_load : memory_store; + } + + bool has_dst = false; + + switch (get_opcode()) { +#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ + case OP: \ + has_dst = (DST != 0); \ + break; +#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ + case OP: \ + has_dst = (DST != 0); \ + break; #include "opcodes.def" #undef OP_DEF #undef OP_W_DEF - default: - printf( "Execution error: Invalid opcode (0x%x)\n", get_opcode() ); + default: + printf("Execution error: Invalid opcode (0x%x)\n", get_opcode()); break; - } + } - switch( m_cache_option ) { - case CA_OPTION: cache_op = CACHE_ALL; break; - case NC_OPTION: cache_op = CACHE_L1; break; - case CG_OPTION: cache_op = CACHE_GLOBAL; break; - case CS_OPTION: cache_op = CACHE_STREAMING; break; - case LU_OPTION: cache_op = CACHE_LAST_USE; break; - case CV_OPTION: cache_op = CACHE_VOLATILE; break; - case WB_OPTION: cache_op = CACHE_WRITE_BACK; break; - case WT_OPTION: cache_op = CACHE_WRITE_THROUGH; break; - default: - //if( m_opcode == LD_OP || m_opcode == LDU_OP ) - if( m_opcode == MMA_LD_OP || m_opcode == LD_OP || m_opcode == LDU_OP ) - cache_op = CACHE_ALL; - //else if( m_opcode == ST_OP ) - else if( m_opcode == MMA_ST_OP || m_opcode == ST_OP ) - cache_op = CACHE_WRITE_BACK; - else if( m_opcode == ATOM_OP ) - cache_op = CACHE_GLOBAL; + switch (m_cache_option) { + case CA_OPTION: + cache_op = CACHE_ALL; break; - } - - set_opcode_and_latency(); - set_bar_type(); - // Get register operands - int n=0,m=0; - ptx_instruction::const_iterator opr=op_iter_begin(); - for ( ; opr != op_iter_end(); opr++, n++ ) { //process operands - const operand_info &o = *opr; - if ( has_dst && n==0 ) { - // Do not set the null register "_" as an architectural register - if ( o.is_reg() && !o.is_non_arch_reg() ) { - out[0] = o.reg_num(); - arch_reg.dst[0] = o.arch_reg_num(); - } else if ( o.is_vector() ) { - is_vectorin = 1; - unsigned num_elem = o.get_vect_nelem(); - if( num_elem >= 1 ) out[0] = o.reg1_num(); - if( num_elem >= 2 ) out[1] = o.reg2_num(); - if( num_elem >= 3 ) out[2] = o.reg3_num(); - if( num_elem >= 4 ) out[3] = o.reg4_num(); - if( num_elem >= 5 ) out[4] = o.reg5_num(); - if( num_elem >= 6 ) out[5] = o.reg6_num(); - if( num_elem >= 7 ) out[6] = o.reg7_num(); - if( num_elem >= 8 ) out[7] = o.reg8_num(); - for (int i = 0; i < num_elem; i++) - arch_reg.dst[i] = o.arch_reg_num(i); - } - } else { - if ( o.is_reg() && !o.is_non_arch_reg() ) { - int reg_num = o.reg_num(); - arch_reg.src[m] = o.arch_reg_num(); - switch ( m ) { - case 0: in[0] = reg_num; break; - case 1: in[1] = reg_num; break; - case 2: in[2] = reg_num; break; - default: break; - } - m++; - } else if ( o.is_vector() ) { - //assert(m == 0); //only support 1 vector operand (for textures) right now - is_vectorout = 1; - unsigned num_elem = o.get_vect_nelem(); - if( num_elem >= 1 ) in[m+0] = o.reg1_num(); - if( num_elem >= 2 ) in[m+1] = o.reg2_num(); - if( num_elem >= 3 ) in[m+2] = o.reg3_num(); - if( num_elem >= 4 ) in[m+3] = o.reg4_num(); - if( num_elem >= 5 ) in[m+4] = o.reg5_num(); - if( num_elem >= 6 ) in[m+5] = o.reg6_num(); - if( num_elem >= 7 ) in[m+6] = o.reg7_num(); - if( num_elem >= 8 ) in[m+7] = o.reg8_num(); - for (int i = 0; i < num_elem; i++) - arch_reg.src[m+i] = o.arch_reg_num(i); - m+=num_elem; - } + case NC_OPTION: + cache_op = CACHE_L1; + break; + case CG_OPTION: + cache_op = CACHE_GLOBAL; + break; + case CS_OPTION: + cache_op = CACHE_STREAMING; + break; + case LU_OPTION: + cache_op = CACHE_LAST_USE; + break; + case CV_OPTION: + cache_op = CACHE_VOLATILE; + break; + case WB_OPTION: + cache_op = CACHE_WRITE_BACK; + break; + case WT_OPTION: + cache_op = CACHE_WRITE_THROUGH; + break; + default: + // if( m_opcode == LD_OP || m_opcode == LDU_OP ) + if (m_opcode == MMA_LD_OP || m_opcode == LD_OP || m_opcode == LDU_OP) + cache_op = CACHE_ALL; + // else if( m_opcode == ST_OP ) + else if (m_opcode == MMA_ST_OP || m_opcode == ST_OP) + cache_op = CACHE_WRITE_BACK; + else if (m_opcode == ATOM_OP) + cache_op = CACHE_GLOBAL; + break; + } + + set_opcode_and_latency(); + set_bar_type(); + // Get register operands + int n = 0, m = 0; + ptx_instruction::const_iterator opr = op_iter_begin(); + for (; opr != op_iter_end(); opr++, n++) { // process operands + const operand_info &o = *opr; + if (has_dst && n == 0) { + // Do not set the null register "_" as an architectural register + if (o.is_reg() && !o.is_non_arch_reg()) { + out[0] = o.reg_num(); + arch_reg.dst[0] = o.arch_reg_num(); + } else if (o.is_vector()) { + is_vectorin = 1; + unsigned num_elem = o.get_vect_nelem(); + if (num_elem >= 1) out[0] = o.reg1_num(); + if (num_elem >= 2) out[1] = o.reg2_num(); + if (num_elem >= 3) out[2] = o.reg3_num(); + if (num_elem >= 4) out[3] = o.reg4_num(); + if (num_elem >= 5) out[4] = o.reg5_num(); + if (num_elem >= 6) out[5] = o.reg6_num(); + if (num_elem >= 7) out[6] = o.reg7_num(); + if (num_elem >= 8) out[7] = o.reg8_num(); + for (int i = 0; i < num_elem; i++) arch_reg.dst[i] = o.arch_reg_num(i); } - } - - //Setting number of input and output operands which is required for scoreboard check - for(int i=0;i0) - outcount++; - - for(int i=0;i0) - incount++; - - // Get predicate - if(has_pred()) { - const operand_info &p = get_pred(); - pred = p.reg_num(); - } - - // Get address registers inside memory operands. - // Assuming only one memory operand per instruction, - // and maximum of two address registers for one memory operand. - if( has_memory_read() || has_memory_write() ) { - ptx_instruction::const_iterator op=op_iter_begin(); - for ( ; op != op_iter_end(); op++, n++ ) { //process operands - const operand_info &o = *op; - - if(o.is_memory_operand()) { - // We do not support the null register as a memory operand - assert( !o.is_non_arch_reg() ); - - // Check PTXPlus-type operand - // memory operand with addressing (ex. s[0x4] or g[$r1]) - if(o.is_memory_operand2()) { - - // memory operand with one address register (ex. g[$r1+0x4] or s[$r2+=0x4]) - if(o.get_double_operand_type() == 0 || o.get_double_operand_type() == 3){ - ar1 = o.reg_num(); - arch_reg.src[4] = o.arch_reg_num(); - // TODO: address register in $r2+=0x4 should be an output register as well - } - // memory operand with two address register (ex. s[$r1+$r1] or g[$r1+=$r2]) - else if(o.get_double_operand_type() == 1 || o.get_double_operand_type() == 2) { - ar1 = o.reg1_num(); - arch_reg.src[4] = o.arch_reg_num(); - ar2 = o.reg2_num(); - arch_reg.src[5] = o.arch_reg_num(); - // TODO: first address register in $r1+=$r2 should be an output register as well - } - } - else if(o.is_immediate_address()){ - - } - // Regular PTX operand - else if (o.get_symbol()->type()->get_key().is_reg()) { // Memory operand contains a register - ar1 = o.reg_num(); - arch_reg.src[4] = o.arch_reg_num(); - } - - } + } else { + if (o.is_reg() && !o.is_non_arch_reg()) { + int reg_num = o.reg_num(); + arch_reg.src[m] = o.arch_reg_num(); + switch (m) { + case 0: + in[0] = reg_num; + break; + case 1: + in[1] = reg_num; + break; + case 2: + in[2] = reg_num; + break; + default: + break; + } + m++; + } else if (o.is_vector()) { + // assert(m == 0); //only support 1 vector operand (for textures) right + // now + is_vectorout = 1; + unsigned num_elem = o.get_vect_nelem(); + if (num_elem >= 1) in[m + 0] = o.reg1_num(); + if (num_elem >= 2) in[m + 1] = o.reg2_num(); + if (num_elem >= 3) in[m + 2] = o.reg3_num(); + if (num_elem >= 4) in[m + 3] = o.reg4_num(); + if (num_elem >= 5) in[m + 4] = o.reg5_num(); + if (num_elem >= 6) in[m + 5] = o.reg6_num(); + if (num_elem >= 7) in[m + 6] = o.reg7_num(); + if (num_elem >= 8) in[m + 7] = o.reg8_num(); + for (int i = 0; i < num_elem; i++) + arch_reg.src[m + i] = o.arch_reg_num(i); + m += num_elem; } - } - - // get reconvergence pc - reconvergence_pc = gpgpu_ctx->func_sim->get_converge_point(pc); - - m_decoded=true; -} - -void function_info::add_param_name_type_size( unsigned index, std::string name, int type, size_t size, bool ptr, memory_space_t space ) -{ - unsigned parsed_index; - char buffer[2048]; - snprintf(buffer,2048,"%s_param_%%u", m_name.c_str() ); - int ntokens = sscanf(name.c_str(),buffer,&parsed_index); - if( ntokens == 1 ) { - assert( m_ptx_kernel_param_info.find(parsed_index) == m_ptx_kernel_param_info.end() ); - m_ptx_kernel_param_info[parsed_index] = param_info(name, type, size, ptr, space); - } else { - assert( m_ptx_kernel_param_info.find(index) == m_ptx_kernel_param_info.end() ); - m_ptx_kernel_param_info[index] = param_info(name, type, size, ptr, space); - } -} - -void function_info::add_param_data( unsigned argn, struct gpgpu_ptx_sim_arg *args ) -{ - const void *data = args->m_start; - - bool scratchpad_memory_param = false; // Is this parameter in CUDA shared memory or OpenCL local memory - - std::map::iterator i=m_ptx_kernel_param_info.find(argn); - if( i != m_ptx_kernel_param_info.end() ) { - if (i->second.is_ptr_shared()) { - assert(args->m_start == NULL && "OpenCL parameter pointer to local memory must have NULL as value"); - scratchpad_memory_param = true; - } else { - param_t tmp; - tmp.pdata = args->m_start; - tmp.size = args->m_nbytes; - tmp.offset = args->m_offset; - tmp.type = 0; - i->second.add_data(tmp); - i->second.add_offset((unsigned) args->m_offset); + } + } + + // Setting number of input and output operands which is required for + // scoreboard check + for (int i = 0; i < MAX_OUTPUT_VALUES; i++) + if (out[i] > 0) outcount++; + + for (int i = 0; i < MAX_INPUT_VALUES; i++) + if (in[i] > 0) incount++; + + // Get predicate + if (has_pred()) { + const operand_info &p = get_pred(); + pred = p.reg_num(); + } + + // Get address registers inside memory operands. + // Assuming only one memory operand per instruction, + // and maximum of two address registers for one memory operand. + if (has_memory_read() || has_memory_write()) { + ptx_instruction::const_iterator op = op_iter_begin(); + for (; op != op_iter_end(); op++, n++) { // process operands + const operand_info &o = *op; + + if (o.is_memory_operand()) { + // We do not support the null register as a memory operand + assert(!o.is_non_arch_reg()); + + // Check PTXPlus-type operand + // memory operand with addressing (ex. s[0x4] or g[$r1]) + if (o.is_memory_operand2()) { + // memory operand with one address register (ex. g[$r1+0x4] or + // s[$r2+=0x4]) + if (o.get_double_operand_type() == 0 || + o.get_double_operand_type() == 3) { + ar1 = o.reg_num(); + arch_reg.src[4] = o.arch_reg_num(); + // TODO: address register in $r2+=0x4 should be an output register + // as well + } + // memory operand with two address register (ex. s[$r1+$r1] or + // g[$r1+=$r2]) + else if (o.get_double_operand_type() == 1 || + o.get_double_operand_type() == 2) { + ar1 = o.reg1_num(); + arch_reg.src[4] = o.arch_reg_num(); + ar2 = o.reg2_num(); + arch_reg.src[5] = o.arch_reg_num(); + // TODO: first address register in $r1+=$r2 should be an output + // register as well + } + } else if (o.is_immediate_address()) { + } + // Regular PTX operand + else if (o.get_symbol() + ->type() + ->get_key() + .is_reg()) { // Memory operand contains a register + ar1 = o.reg_num(); + arch_reg.src[4] = o.arch_reg_num(); + } } - } else { - scratchpad_memory_param = true; - } - - if (scratchpad_memory_param) { - // This should only happen for OpenCL: - // - // The LLVM PTX compiler in NVIDIA's driver (version 190.29) - // does not generate an argument in the function declaration - // for __constant arguments. - // - // The associated constant memory space can be allocated in two - // ways. It can be explicitly initialized in the .ptx file where - // it is declared. Or, it can be allocated using the clCreateBuffer - // on the host. In this later case, the .ptx file will contain - // a global declaration of the parameter, but it will have an unknown - // array size. Thus, the symbol's address will not be set and we need - // to set it here before executing the PTX. - - char buffer[2048]; - snprintf(buffer,2048,"%s_param_%u",m_name.c_str(),argn); - - symbol *p = m_symtab->lookup(buffer); - if( p == NULL ) { - printf("GPGPU-Sim PTX: ERROR ** could not locate symbol for \'%s\' : cannot bind buffer\n", buffer); - abort(); + } + } + + // get reconvergence pc + reconvergence_pc = gpgpu_ctx->func_sim->get_converge_point(pc); + + m_decoded = true; +} + +void function_info::add_param_name_type_size(unsigned index, std::string name, + int type, size_t size, bool ptr, + memory_space_t space) { + unsigned parsed_index; + char buffer[2048]; + snprintf(buffer, 2048, "%s_param_%%u", m_name.c_str()); + int ntokens = sscanf(name.c_str(), buffer, &parsed_index); + if (ntokens == 1) { + assert(m_ptx_kernel_param_info.find(parsed_index) == + m_ptx_kernel_param_info.end()); + m_ptx_kernel_param_info[parsed_index] = + param_info(name, type, size, ptr, space); + } else { + assert(m_ptx_kernel_param_info.find(index) == + m_ptx_kernel_param_info.end()); + m_ptx_kernel_param_info[index] = param_info(name, type, size, ptr, space); + } +} + +void function_info::add_param_data(unsigned argn, + struct gpgpu_ptx_sim_arg *args) { + const void *data = args->m_start; + + bool scratchpad_memory_param = + false; // Is this parameter in CUDA shared memory or OpenCL local memory + + std::map::iterator i = + m_ptx_kernel_param_info.find(argn); + if (i != m_ptx_kernel_param_info.end()) { + if (i->second.is_ptr_shared()) { + assert( + args->m_start == NULL && + "OpenCL parameter pointer to local memory must have NULL as value"); + scratchpad_memory_param = true; + } else { + param_t tmp; + tmp.pdata = args->m_start; + tmp.size = args->m_nbytes; + tmp.offset = args->m_offset; + tmp.type = 0; + i->second.add_data(tmp); + i->second.add_offset((unsigned)args->m_offset); + } + } else { + scratchpad_memory_param = true; + } + + if (scratchpad_memory_param) { + // This should only happen for OpenCL: + // + // The LLVM PTX compiler in NVIDIA's driver (version 190.29) + // does not generate an argument in the function declaration + // for __constant arguments. + // + // The associated constant memory space can be allocated in two + // ways. It can be explicitly initialized in the .ptx file where + // it is declared. Or, it can be allocated using the clCreateBuffer + // on the host. In this later case, the .ptx file will contain + // a global declaration of the parameter, but it will have an unknown + // array size. Thus, the symbol's address will not be set and we need + // to set it here before executing the PTX. + + char buffer[2048]; + snprintf(buffer, 2048, "%s_param_%u", m_name.c_str(), argn); + + symbol *p = m_symtab->lookup(buffer); + if (p == NULL) { + printf( + "GPGPU-Sim PTX: ERROR ** could not locate symbol for \'%s\' : cannot " + "bind buffer\n", + buffer); + abort(); + } + if (data) + p->set_address((addr_t) * (size_t *)data); + else { + // clSetKernelArg was passed NULL pointer for data... + // this is used for dynamically sized shared memory on NVIDIA platforms + bool is_ptr_shared = false; + if (i != m_ptx_kernel_param_info.end()) { + is_ptr_shared = i->second.is_ptr_shared(); } - if( data ) - p->set_address((addr_t)*(size_t*)data); - else { - // clSetKernelArg was passed NULL pointer for data... - // this is used for dynamically sized shared memory on NVIDIA platforms - bool is_ptr_shared = false; - if( i != m_ptx_kernel_param_info.end() ) { - is_ptr_shared = i->second.is_ptr_shared(); - } - - if( !is_ptr_shared and !p->is_shared() ) { - printf("GPGPU-Sim PTX: ERROR ** clSetKernelArg passed NULL but arg not shared memory\n"); - abort(); - } - unsigned num_bits = 8*args->m_nbytes; - printf("GPGPU-Sim PTX: deferred allocation of shared region for \"%s\" from 0x%x to 0x%x (shared memory space)\n", - p->name().c_str(), - m_symtab->get_shared_next(), - m_symtab->get_shared_next() + num_bits/8 ); - fflush(stdout); - assert( (num_bits%8) == 0 ); - addr_t addr = m_symtab->get_shared_next(); - addr_t addr_pad = num_bits ? (((num_bits/8) - (addr % (num_bits/8))) % (num_bits/8)) : 0; - p->set_address( addr+addr_pad ); - m_symtab->alloc_shared( num_bits/8 + addr_pad ); + + if (!is_ptr_shared and !p->is_shared()) { + printf( + "GPGPU-Sim PTX: ERROR ** clSetKernelArg passed NULL but arg not " + "shared memory\n"); + abort(); } - } + unsigned num_bits = 8 * args->m_nbytes; + printf( + "GPGPU-Sim PTX: deferred allocation of shared region for \"%s\" from " + "0x%x to 0x%x (shared memory space)\n", + p->name().c_str(), m_symtab->get_shared_next(), + m_symtab->get_shared_next() + num_bits / 8); + fflush(stdout); + assert((num_bits % 8) == 0); + addr_t addr = m_symtab->get_shared_next(); + addr_t addr_pad = + num_bits + ? (((num_bits / 8) - (addr % (num_bits / 8))) % (num_bits / 8)) + : 0; + p->set_address(addr + addr_pad); + m_symtab->alloc_shared(num_bits / 8 + addr_pad); + } + } } unsigned function_info::get_args_aligned_size() { - - if(m_args_aligned_size >= 0) - return m_args_aligned_size; - - unsigned param_address = 0; - unsigned int total_size = 0; - for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); - - size_t arg_size = p.get_size() / 8; // size of param in bytes - total_size = (total_size + arg_size - 1) / arg_size * arg_size; //aligned - p.add_offset(total_size); - param->set_address(param_address + total_size); - total_size += arg_size; - } - - m_args_aligned_size = (total_size + 3) / 4 * 4; //final size aligned to word - - return m_args_aligned_size; - -} - - -void function_info::finalize( memory_space *param_mem ) -{ - unsigned param_address = 0; - for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - param_info &p = i->second; - if (p.is_ptr_shared()) continue; // Pointer to local memory: Should we pass the allocated shared memory address to the param memory space? - std::string name = p.get_name(); - int type = p.get_type(); - param_t param_value = p.get_value(); - param_value.type = type; - symbol *param = m_symtab->lookup(name.c_str()); - unsigned xtype = param->type()->get_key().scalar_type(); - assert(xtype==(unsigned)type); - size_t size; - size = param_value.size; // size of param in bytes - // assert(param_value.offset == param_address); - if( size != p.get_size() / 8) { - printf("GPGPU-Sim PTX: WARNING actual kernel paramter size = %zu bytes vs. formal size = %zu (using smaller of two)\n", - size, p.get_size()/8); - size = (size<(p.get_size()/8))?size:(p.get_size()/8); - } - // copy the parameter over word-by-word so that parameter that crosses a memory page can be copied over - //Jin: copy parameter using aligned rules - const type_info *paramtype = param->type(); - int align_amount = paramtype->get_key().get_alignment_spec(); - align_amount = (align_amount == -1) ? size : align_amount; - param_address = (param_address + align_amount - 1) / align_amount * align_amount; //aligned - - const size_t word_size = 4; - //param_address = (param_address + size - 1) / size * size; //aligned with size - for (size_t idx = 0; idx < size; idx += word_size) { - const char *pdata = reinterpret_cast(param_value.pdata) + idx; // cast to char * for ptr arithmetic - param_mem->write(param_address + idx, word_size, pdata,NULL,NULL); + if (m_args_aligned_size >= 0) return m_args_aligned_size; + + unsigned param_address = 0; + unsigned int total_size = 0; + for (std::map::iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + + size_t arg_size = p.get_size() / 8; // size of param in bytes + total_size = (total_size + arg_size - 1) / arg_size * arg_size; // aligned + p.add_offset(total_size); + param->set_address(param_address + total_size); + total_size += arg_size; + } + + m_args_aligned_size = (total_size + 3) / 4 * 4; // final size aligned to word + + return m_args_aligned_size; +} + +void function_info::finalize(memory_space *param_mem) { + unsigned param_address = 0; + for (std::map::iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + param_info &p = i->second; + if (p.is_ptr_shared()) + continue; // Pointer to local memory: Should we pass the allocated shared + // memory address to the param memory space? + std::string name = p.get_name(); + int type = p.get_type(); + param_t param_value = p.get_value(); + param_value.type = type; + symbol *param = m_symtab->lookup(name.c_str()); + unsigned xtype = param->type()->get_key().scalar_type(); + assert(xtype == (unsigned)type); + size_t size; + size = param_value.size; // size of param in bytes + // assert(param_value.offset == param_address); + if (size != p.get_size() / 8) { + printf( + "GPGPU-Sim PTX: WARNING actual kernel paramter size = %zu bytes vs. " + "formal size = %zu (using smaller of two)\n", + size, p.get_size() / 8); + size = (size < (p.get_size() / 8)) ? size : (p.get_size() / 8); + } + // copy the parameter over word-by-word so that parameter that crosses a + // memory page can be copied over + // Jin: copy parameter using aligned rules + const type_info *paramtype = param->type(); + int align_amount = paramtype->get_key().get_alignment_spec(); + align_amount = (align_amount == -1) ? size : align_amount; + param_address = (param_address + align_amount - 1) / align_amount * + align_amount; // aligned + + const size_t word_size = 4; + // param_address = (param_address + size - 1) / size * size; //aligned with + // size + for (size_t idx = 0; idx < size; idx += word_size) { + const char *pdata = reinterpret_cast(param_value.pdata) + + idx; // cast to char * for ptr arithmetic + param_mem->write(param_address + idx, word_size, pdata, NULL, NULL); + } + unsigned offset = p.get_offset(); + assert(offset == param_address); + param->set_address(param_address); + param_address += size; + } +} + +void function_info::param_to_shared(memory_space *shared_mem, + symbol_table *symtab) { + // TODO: call this only for PTXPlus with GT200 models + // extern gpgpu_sim* g_the_gpu; + if (not gpgpu_ctx->the_gpgpusim->g_the_gpu->get_config().convert_to_ptxplus()) + return; + + // copies parameters into simulated shared memory + for (std::map::iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + param_info &p = i->second; + if (p.is_ptr_shared()) + continue; // Pointer to local memory: Should we pass the allocated shared + // memory address to the param memory space? + std::string name = p.get_name(); + int type = p.get_type(); + param_t value = p.get_value(); + value.type = type; + symbol *param = symtab->lookup(name.c_str()); + unsigned xtype = param->type()->get_key().scalar_type(); + assert(xtype == (unsigned)type); + + int tmp; + size_t size; + unsigned offset = p.get_offset(); + type_info_key::type_decode(xtype, size, tmp); + + // Write to shared memory - offset + 0x10 + shared_mem->write(offset + 0x10, size / 8, value.pdata, NULL, NULL); + } +} + +void function_info::list_param(FILE *fout) const { + for (std::map::const_iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + const param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); + fprintf(fout, "%s: %#08x\n", name.c_str(), param_addr); + } + fflush(fout); +} + +void function_info::ptx_jit_config( + std::map mallocPtr_Size, + memory_space *param_mem, gpgpu_t *gpu, dim3 gridDim, dim3 blockDim) { + static unsigned long long counter = 0; + std::vector > param_data; + std::vector offsets; + std::vector paramIsPointer; + + char *gpgpusim_path = getenv("GPGPUSIM_ROOT"); + assert(gpgpusim_path != NULL); + char *wys_exec_path = getenv("WYS_EXEC_PATH"); + assert(wys_exec_path != NULL); + std::string command = + std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; + std::string filename(std::string(gpgpusim_path) + + "/debug_tools/WatchYourStep/data/params.config" + + std::to_string(counter)); + + // initialize paramList + char buff[1024]; + std::string filename_c(filename + "_c"); + snprintf(buff, 1024, "c++filt %s > %s", get_name().c_str(), + filename_c.c_str()); + assert(system(buff) != NULL); + FILE *fp = fopen(filename_c.c_str(), "r"); + fgets(buff, 1024, fp); + fclose(fp); + std::string fn(buff); + size_t pos1, pos2; + pos1 = fn.find_last_of("("); + pos2 = fn.find(")", pos1); + assert(pos2 > pos1 && pos1 > 0); + strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); + char *tok; + tok = strtok(buff, ","); + std::string tmp; + while (tok != NULL) { + std::string param(tok); + if (param.find("<") != std::string::npos) { + assert(param.find(">") == std::string::npos); + assert(param.find("*") == std::string::npos); + tmp = param; + } else { + if (tmp.length() > 0) { + tmp = ""; + assert(param.find(">") != std::string::npos); + assert(param.find("<") == std::string::npos); + assert(param.find("*") == std::string::npos); + } + printf("%s\n", param.c_str()); + if (param.find("*") != std::string::npos) { + paramIsPointer.push_back(true); + } else { + paramIsPointer.push_back(false); } - unsigned offset = p.get_offset(); - assert(offset == param_address); - param->set_address(param_address); - param_address += size; - } -} - -void function_info::param_to_shared( memory_space *shared_mem, symbol_table *symtab ) -{ - // TODO: call this only for PTXPlus with GT200 models - //extern gpgpu_sim* g_the_gpu; - if (not gpgpu_ctx->the_gpgpusim->g_the_gpu->get_config().convert_to_ptxplus()) return; - - // copies parameters into simulated shared memory - for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - param_info &p = i->second; - if (p.is_ptr_shared()) continue; // Pointer to local memory: Should we pass the allocated shared memory address to the param memory space? - std::string name = p.get_name(); - int type = p.get_type(); - param_t value = p.get_value(); - value.type = type; - symbol *param = symtab->lookup(name.c_str()); - unsigned xtype = param->type()->get_key().scalar_type(); - assert(xtype==(unsigned)type); - - int tmp; - size_t size; - unsigned offset = p.get_offset(); - type_info_key::type_decode(xtype,size,tmp); - - // Write to shared memory - offset + 0x10 - shared_mem->write(offset+0x10,size/8,value.pdata,NULL,NULL); - } -} - - -void function_info::list_param( FILE *fout ) const -{ - for( std::map::const_iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - const param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); - addr_t param_addr = param->get_address(); - fprintf(fout, "%s: %#08x\n", name.c_str(), param_addr); - } - fflush(fout); -} - -void function_info::ptx_jit_config(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) -{ - static unsigned long long counter = 0; - std::vector< std::pair > param_data; - std::vector offsets; - std::vector paramIsPointer; - - char * gpgpusim_path = getenv("GPGPUSIM_ROOT"); - assert(gpgpusim_path!=NULL); - char * wys_exec_path = getenv("WYS_EXEC_PATH"); - assert(wys_exec_path!=NULL); - std::string command = std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; - std::string filename(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/params.config" + std::to_string(counter)); - - //initialize paramList - char buff[1024]; - std::string filename_c(filename+"_c"); - snprintf(buff,1024,"c++filt %s > %s", get_name().c_str(), filename_c.c_str()); - assert(system(buff) != NULL); - FILE *fp = fopen(filename_c.c_str(), "r"); - fgets(buff, 1024, fp); - fclose(fp); - std::string fn(buff); - size_t pos1, pos2; - pos1 = fn.find_last_of("("); - pos2 = fn.find(")", pos1); - assert(pos2>pos1&&pos1>0); - strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); - char *tok; - tok = strtok(buff, ","); - std::string tmp; - while(tok!=NULL){ - std::string param(tok); - if(param.find("<")!=std::string::npos){ - assert(param.find(">")==std::string::npos); - assert(param.find("*")==std::string::npos); - tmp = param; - } else { - if (tmp.length()>0){ - tmp = ""; - assert(param.find(">")!=std::string::npos); - assert(param.find("<")==std::string::npos); - assert(param.find("*")==std::string::npos); - } - printf("%s\n", param.c_str()); - if(param.find("*")!=std::string::npos){ - paramIsPointer.push_back(true); - }else{ - paramIsPointer.push_back(false); - } - } - tok = strtok(NULL, ","); } - - - for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); - addr_t param_addr = param->get_address(); - param_t param_value = p.get_value(); - offsets.push_back((unsigned)p.get_offset()); - - if (paramIsPointer[i->first] && (*(unsigned long long*)param_value.pdata != 0)){ - //is pointer - assert(param_value.size==sizeof(void*)&&"MisID'd this param as pointer"); - size_t array_size = 0; - unsigned long long param_pointer = *(unsigned long long*)param_value.pdata; - if(mallocPtr_Size.find(param_pointer)!=mallocPtr_Size.end()){ - array_size = mallocPtr_Size[param_pointer]; - }else{ - for( std::map::iterator j=mallocPtr_Size.begin(); j!=mallocPtr_Size.end(); j++ ) { - if(param_pointer>j->first&¶m_pointerfirst + j->second){ - array_size = j->first + j->second - param_pointer; - break; - } - } - assert(array_size>0&&"pointer was not previously malloc'd"); - } - - unsigned char* val = (unsigned char*) malloc(param_value.size); - param_mem->read(param_addr,param_value.size,(void*)val); - unsigned char* array_val = (unsigned char*) malloc(array_size); - gpu->get_global_memory()->read(*(unsigned*)((void*)val),array_size,(void*)array_val); - param_data.push_back(std::pair(array_size,array_val)); - paramIsPointer.push_back(true); - }else{ - unsigned char* val = (unsigned char*) malloc(param_value.size); - param_mem->read(param_addr,param_value.size,(void*)val); - param_data.push_back(std::pair(param_value.size,val)); - paramIsPointer.push_back(false); + tok = strtok(NULL, ","); + } + + for (std::map::iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); + param_t param_value = p.get_value(); + offsets.push_back((unsigned)p.get_offset()); + + if (paramIsPointer[i->first] && + (*(unsigned long long *)param_value.pdata != 0)) { + // is pointer + assert(param_value.size == sizeof(void *) && + "MisID'd this param as pointer"); + size_t array_size = 0; + unsigned long long param_pointer = + *(unsigned long long *)param_value.pdata; + if (mallocPtr_Size.find(param_pointer) != mallocPtr_Size.end()) { + array_size = mallocPtr_Size[param_pointer]; + } else { + for (std::map::iterator j = + mallocPtr_Size.begin(); + j != mallocPtr_Size.end(); j++) { + if (param_pointer > j->first && + param_pointer < j->first + j->second) { + array_size = j->first + j->second - param_pointer; + break; + } } - } + assert(array_size > 0 && "pointer was not previously malloc'd"); + } - FILE *fout = fopen (filename.c_str(), "w"); - printf("Writing data to %s ...\n", filename.c_str()); - fprintf(fout, "%s\n", get_name().c_str()); - fprintf(fout, "%u,%u,%u %u,%u,%u\n", gridDim.x, gridDim.y, gridDim.z, blockDim.x, blockDim.y, blockDim.z); - size_t index = 0; - for( std::vector< std::pair >::const_iterator i=param_data.begin(); i!=param_data.end(); i++ ) { - if (paramIsPointer[index]){ - fprintf(fout, "*"); - } - fprintf(fout, "%lu :", i->first); - for (size_t j = 0; jfirst; j++){ - fprintf(fout, " %u", i->second[j]); - } - fprintf(fout, " : %u", offsets[index]); - free (i->second); - fprintf(fout, "\n"); - index++; + unsigned char *val = (unsigned char *)malloc(param_value.size); + param_mem->read(param_addr, param_value.size, (void *)val); + unsigned char *array_val = (unsigned char *)malloc(array_size); + gpu->get_global_memory()->read(*(unsigned *)((void *)val), array_size, + (void *)array_val); + param_data.push_back( + std::pair(array_size, array_val)); + paramIsPointer.push_back(true); + } else { + unsigned char *val = (unsigned char *)malloc(param_value.size); + param_mem->read(param_addr, param_value.size, (void *)val); + param_data.push_back( + std::pair(param_value.size, val)); + paramIsPointer.push_back(false); } - fflush(fout); - fclose(fout); - - //ptx config - std::string ptx_config_fn(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/ptx.config" + std::to_string(counter)); - snprintf(buff, 1024, "grep -rn \".entry %s\" %s/*.ptx | cut -d \":\" -f 1-2 > %s", get_name().c_str(), wys_exec_path, ptx_config_fn.c_str()); - if (system(buff)!=0){ - printf("WARNING: Failed to execute grep to find ptx source \n"); - printf("Problematic call: %s", buff); - abort(); + } + + FILE *fout = fopen(filename.c_str(), "w"); + printf("Writing data to %s ...\n", filename.c_str()); + fprintf(fout, "%s\n", get_name().c_str()); + fprintf(fout, "%u,%u,%u %u,%u,%u\n", gridDim.x, gridDim.y, gridDim.z, + blockDim.x, blockDim.y, blockDim.z); + size_t index = 0; + for (std::vector >::const_iterator i = + param_data.begin(); + i != param_data.end(); i++) { + if (paramIsPointer[index]) { + fprintf(fout, "*"); } - FILE *fin = fopen(ptx_config_fn.c_str(), "r"); - char ptx_source[256]; - unsigned line_number; - int numscanned = fscanf(fin, "%[^:]:%u", ptx_source, &line_number); - assert(numscanned == 2); - fclose(fin); - snprintf(buff, 1024, "grep -rn \".version\" %s | cut -d \":\" -f 1 | xargs -I \"{}\" awk \"NR>={}&&NR<={}+2\" %s > %s", ptx_source, ptx_source, ptx_config_fn.c_str()); - if (system(buff)!=0){ - printf("WARNING: Failed to execute grep to find ptx header \n"); - printf("Problematic call: %s", buff); - abort(); + fprintf(fout, "%lu :", i->first); + for (size_t j = 0; j < i->first; j++) { + fprintf(fout, " %u", i->second[j]); } - fin = fopen(ptx_source, "r"); - assert(fin!=NULL); - printf("Writing data to %s ...\n", ptx_config_fn.c_str()); - fout = fopen(ptx_config_fn.c_str(), "a"); - assert(fout!=NULL); - for (unsigned i = 0; isecond); + fprintf(fout, "\n"); + index++; + } + fflush(fout); + fclose(fout); + + // ptx config + std::string ptx_config_fn(std::string(gpgpusim_path) + + "/debug_tools/WatchYourStep/data/ptx.config" + + std::to_string(counter)); + snprintf(buff, 1024, + "grep -rn \".entry %s\" %s/*.ptx | cut -d \":\" -f 1-2 > %s", + get_name().c_str(), wys_exec_path, ptx_config_fn.c_str()); + if (system(buff) != 0) { + printf("WARNING: Failed to execute grep to find ptx source \n"); + printf("Problematic call: %s", buff); + abort(); + } + FILE *fin = fopen(ptx_config_fn.c_str(), "r"); + char ptx_source[256]; + unsigned line_number; + int numscanned = fscanf(fin, "%[^:]:%u", ptx_source, &line_number); + assert(numscanned == 2); + fclose(fin); + snprintf(buff, 1024, + "grep -rn \".version\" %s | cut -d \":\" -f 1 | xargs -I \"{}\" awk " + "\"NR>={}&&NR<={}+2\" %s > %s", + ptx_source, ptx_source, ptx_config_fn.c_str()); + if (system(buff) != 0) { + printf("WARNING: Failed to execute grep to find ptx header \n"); + printf("Problematic call: %s", buff); + abort(); + } + fin = fopen(ptx_source, "r"); + assert(fin != NULL); + printf("Writing data to %s ...\n", ptx_config_fn.c_str()); + fout = fopen(ptx_config_fn.c_str(), "a"); + assert(fout != NULL); + for (unsigned i = 0; i < line_number; i++) { + assert(fgets(buff, 1024, fin) != NULL); + assert(!feof(fin)); + } + fprintf(fout, "\n\n"); + do { + fprintf(fout, "%s", buff); + assert(fgets(buff, 1024, fin) != NULL); + if (feof(fin)) { + break; } - fprintf(fout, "\n\n"); - do{ - fprintf(fout, "%s", buff); - assert(fgets(buff, 1024, fin) != NULL); - if(feof(fin)){ - break; - } - } while(strstr(buff, "entry")==NULL); + } while (strstr(buff, "entry") == NULL); - fclose(fin); - fflush(fout); - fclose(fout); - counter++; + fclose(fin); + fflush(fout); + fclose(fout); + counter++; } -template -bool cuda_sim::ptx_debug_exec_dump_cond(int thd_uid, addr_t pc) -{ - if (g_debug_execution >= activate_level) { - // check each type of debug dump constraint to filter out dumps - if ( (g_debug_thread_uid != 0) && (thd_uid != (unsigned)g_debug_thread_uid) ) { - return false; - } - if ( (g_debug_pc != 0xBEEF1518) && (pc != g_debug_pc) ) { - return false; - } +template +bool cuda_sim::ptx_debug_exec_dump_cond(int thd_uid, addr_t pc) { + if (g_debug_execution >= activate_level) { + // check each type of debug dump constraint to filter out dumps + if ((g_debug_thread_uid != 0) && + (thd_uid != (unsigned)g_debug_thread_uid)) { + return false; + } + if ((g_debug_pc != 0xBEEF1518) && (pc != g_debug_pc)) { + return false; + } - return true; - } - - return false; -} - -void cuda_sim::init_inst_classification_stat() -{ - static std::set init; - if( init.find(g_ptx_kernel_count) != init.end() ) - return; - init.insert(g_ptx_kernel_count); - - #define MAX_CLASS_KER 1024 - char kernelname[MAX_CLASS_KER] =""; - if (!g_inst_classification_stat) g_inst_classification_stat = (void**)calloc(MAX_CLASS_KER, sizeof(void*)); - snprintf(kernelname, MAX_CLASS_KER, "Kernel %d Classification\n",g_ptx_kernel_count ); - assert( g_ptx_kernel_count < MAX_CLASS_KER ) ; // a static limit on number of kernels increase it if it fails! - g_inst_classification_stat[g_ptx_kernel_count] = StatCreate(kernelname,1,20); - if (!g_inst_op_classification_stat) g_inst_op_classification_stat = (void**)calloc(MAX_CLASS_KER, sizeof(void*)); - snprintf(kernelname, MAX_CLASS_KER, "Kernel %d OP Classification\n",g_ptx_kernel_count ); - g_inst_op_classification_stat[g_ptx_kernel_count] = StatCreate(kernelname,1,100); -} - -static unsigned get_tex_datasize( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &src1 = pI->src1(); //the name of the texture - std::string texname = src1.name(); - - /* - For programs with many streams, textures can be bound and unbound - asynchronously. This means we need to use the kernel's "snapshot" of - the state of the texture mappings when it was launched (so that we - don't try to access the incorrect texture mapping if it's been updated, - or that we don't access a mapping that has been unbound). - */ - kernel_info_t& k = thread->get_kernel(); - const struct textureInfo* texInfo = k.get_texinfo(texname); - - unsigned data_size = texInfo->texel_size; - return data_size; -} - -int tensorcore_op(int inst_opcode){ - - if((inst_opcode==MMA_OP)||(inst_opcode==MMA_LD_OP)||(inst_opcode==MMA_ST_OP)) - return 1; - else - return 0; -} -void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id) -{ - - bool skip = false; - int op_classification = 0; - addr_t pc = next_instr(); - assert( pc == inst.pc ); // make sure timing model and functional model are in sync - const ptx_instruction *pI = m_func_info->get_instruction(pc); - - set_npc( pc + pI->inst_size() ); - - - try { - - clearRPC(); - m_last_set_operand_value.u64 = 0; - - if(is_done()) - { - printf("attempted to execute instruction on a thread that is already done.\n"); + return true; + } + + return false; +} + +void cuda_sim::init_inst_classification_stat() { + static std::set init; + if (init.find(g_ptx_kernel_count) != init.end()) return; + init.insert(g_ptx_kernel_count); + +#define MAX_CLASS_KER 1024 + char kernelname[MAX_CLASS_KER] = ""; + if (!g_inst_classification_stat) + g_inst_classification_stat = (void **)calloc(MAX_CLASS_KER, sizeof(void *)); + snprintf(kernelname, MAX_CLASS_KER, "Kernel %d Classification\n", + g_ptx_kernel_count); + assert(g_ptx_kernel_count < MAX_CLASS_KER); // a static limit on number of + // kernels increase it if it + // fails! + g_inst_classification_stat[g_ptx_kernel_count] = + StatCreate(kernelname, 1, 20); + if (!g_inst_op_classification_stat) + g_inst_op_classification_stat = + (void **)calloc(MAX_CLASS_KER, sizeof(void *)); + snprintf(kernelname, MAX_CLASS_KER, "Kernel %d OP Classification\n", + g_ptx_kernel_count); + g_inst_op_classification_stat[g_ptx_kernel_count] = + StatCreate(kernelname, 1, 100); +} + +static unsigned get_tex_datasize(const ptx_instruction *pI, + ptx_thread_info *thread) { + const operand_info &src1 = pI->src1(); // the name of the texture + std::string texname = src1.name(); + + /* + For programs with many streams, textures can be bound and unbound + asynchronously. This means we need to use the kernel's "snapshot" of + the state of the texture mappings when it was launched (so that we + don't try to access the incorrect texture mapping if it's been updated, + or that we don't access a mapping that has been unbound). + */ + kernel_info_t &k = thread->get_kernel(); + const struct textureInfo *texInfo = k.get_texinfo(texname); + + unsigned data_size = texInfo->texel_size; + return data_size; +} + +int tensorcore_op(int inst_opcode) { + if ((inst_opcode == MMA_OP) || (inst_opcode == MMA_LD_OP) || + (inst_opcode == MMA_ST_OP)) + return 1; + else + return 0; +} +void ptx_thread_info::ptx_exec_inst(warp_inst_t &inst, unsigned lane_id) { + bool skip = false; + int op_classification = 0; + addr_t pc = next_instr(); + assert(pc == + inst.pc); // make sure timing model and functional model are in sync + const ptx_instruction *pI = m_func_info->get_instruction(pc); + + set_npc(pc + pI->inst_size()); + + try { + clearRPC(); + m_last_set_operand_value.u64 = 0; + + if (is_done()) { + printf( + "attempted to execute instruction on a thread that is already " + "done.\n"); assert(0); - } - - if ( g_debug_execution >= 6 || m_gpu->get_config().get_ptx_inst_debug_to_file()) { - if ( (m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid==0) - || (get_uid() == (unsigned)(m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid)) ) { - - clear_modifiedregs(); - enable_debug_trace(); + } + + if (g_debug_execution >= 6 || + m_gpu->get_config().get_ptx_inst_debug_to_file()) { + if ((m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid == 0) || + (get_uid() == + (unsigned)(m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid))) { + clear_modifiedregs(); + enable_debug_trace(); } - } - - - if( pI->has_pred() ) { + } + + if (pI->has_pred()) { const operand_info &pred = pI->get_pred(); ptx_reg_t pred_value = get_operand_value(pred, pred, PRED_TYPE, this, 0); - if(pI->get_pred_mod() == -1) { - skip = (pred_value.pred & 0x0001) ^ pI->get_pred_neg(); //ptxplus inverts the zero flag + if (pI->get_pred_mod() == -1) { + skip = (pred_value.pred & 0x0001) ^ + pI->get_pred_neg(); // ptxplus inverts the zero flag } else { - skip = !pred_lookup(pI->get_pred_mod(), pred_value.pred & 0x000F); + skip = !pred_lookup(pI->get_pred_mod(), pred_value.pred & 0x000F); } - } - int inst_opcode=pI->get_opcode(); - - if( skip ) { + } + int inst_opcode = pI->get_opcode(); + + if (skip) { inst.set_not_active(lane_id); - } else { + } else { const ptx_instruction *pI_saved = pI; ptx_instruction *pJ = NULL; - if( pI->get_opcode() == VOTE_OP ) { - pJ = new ptx_instruction(*pI); - *((warp_inst_t*)pJ) = inst; // copy active mask information - pI = pJ; + if (pI->get_opcode() == VOTE_OP) { + pJ = new ptx_instruction(*pI); + *((warp_inst_t *)pJ) = inst; // copy active mask information + pI = pJ; } - - if(((inst_opcode==MMA_OP||inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))){ - if(inst.active_count()!=MAX_WARP_SIZE) - { - printf("Tensor Core operation are warp synchronous operation. All the threads needs to be active."); - assert(0); - } + + if (((inst_opcode == MMA_OP || inst_opcode == MMA_LD_OP || + inst_opcode == MMA_ST_OP))) { + if (inst.active_count() != MAX_WARP_SIZE) { + printf( + "Tensor Core operation are warp synchronous operation. All the " + "threads needs to be active."); + assert(0); + } } - - //Tensorcore is warp synchronous operation. So these instructions needs to be executed only once. To make the simulation faster removing the redundant tensorcore operation - if(!tensorcore_op(inst_opcode)||((tensorcore_op(inst_opcode))&&(lane_id==0))){ - switch ( inst_opcode ) { - #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,this); op_classification = CLASSIFICATION; break; - #define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,get_core(),inst); op_classification = CLASSIFICATION; break; - #include "opcodes.def" - #undef OP_DEF - #undef OP_W_DEF - default: printf( "Execution error: Invalid opcode (0x%x)\n", pI->get_opcode() ); break; - } + + // Tensorcore is warp synchronous operation. So these instructions needs + // to be executed only once. To make the simulation faster removing the + // redundant tensorcore operation + if (!tensorcore_op(inst_opcode) || + ((tensorcore_op(inst_opcode)) && (lane_id == 0))) { + switch (inst_opcode) { +#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ + case OP: \ + FUNC(pI, this); \ + op_classification = CLASSIFICATION; \ + break; +#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ + case OP: \ + FUNC(pI, get_core(), inst); \ + op_classification = CLASSIFICATION; \ + break; +#include "opcodes.def" +#undef OP_DEF +#undef OP_W_DEF + default: + printf("Execution error: Invalid opcode (0x%x)\n", + pI->get_opcode()); + break; + } } delete pJ; pI = pI_saved; - + // Run exit instruction if exit option included - if(pI->is_exit()) - exit_impl(pI,this); - } - - - - const gpgpu_functional_sim_config &config = m_gpu->get_config(); - - // Output instruction information to file and stdout - if( config.get_ptx_inst_debug_to_file() != 0 && - (config.get_ptx_inst_debug_thread_uid() == 0 || config.get_ptx_inst_debug_thread_uid() == get_uid()) ) { - fprintf(m_gpu->get_ptx_inst_debug_file(), - "[thd=%u] : (%s:%u - %s)\n", - get_uid(), - pI->source_file(), pI->source_line(), pI->get_source() ); - //fprintf(ptx_inst_debug_file, "has memory read=%d, has memory write=%d\n", pI->has_memory_read(), pI->has_memory_write()); + if (pI->is_exit()) exit_impl(pI, this); + } + + const gpgpu_functional_sim_config &config = m_gpu->get_config(); + + // Output instruction information to file and stdout + if (config.get_ptx_inst_debug_to_file() != 0 && + (config.get_ptx_inst_debug_thread_uid() == 0 || + config.get_ptx_inst_debug_thread_uid() == get_uid())) { + fprintf(m_gpu->get_ptx_inst_debug_file(), "[thd=%u] : (%s:%u - %s)\n", + get_uid(), pI->source_file(), pI->source_line(), + pI->get_source()); + // fprintf(ptx_inst_debug_file, "has memory read=%d, has memory + // write=%d\n", pI->has_memory_read(), pI->has_memory_write()); fflush(m_gpu->get_ptx_inst_debug_file()); - } + } - if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<5>(get_uid(), pc) ) { + if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<5>(get_uid(), + pc)) { dim3 ctaid = get_ctaid(); dim3 tid = get_tid(); - printf("%u [thd=%u][i=%u] : ctaid=(%u,%u,%u) tid=(%u,%u,%u) icount=%u [pc=%u] (%s:%u - %s) [0x%llx]\n", - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, - get_uid(), - pI->uid(), ctaid.x,ctaid.y,ctaid.z,tid.x,tid.y,tid.z, - get_icount(), - pc, pI->source_file(), pI->source_line(), pI->get_source(), - m_last_set_operand_value.u64 ); + printf( + "%u [thd=%u][i=%u] : ctaid=(%u,%u,%u) tid=(%u,%u,%u) icount=%u " + "[pc=%u] (%s:%u - %s) [0x%llx]\n", + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, get_uid(), pI->uid(), + ctaid.x, ctaid.y, ctaid.z, tid.x, tid.y, tid.z, get_icount(), pc, + pI->source_file(), pI->source_line(), pI->get_source(), + m_last_set_operand_value.u64); fflush(stdout); - } - - addr_t insn_memaddr = 0xFEEBDAED; - memory_space_t insn_space = undefined_space; - _memory_op_t insn_memory_op = no_memory_op; - unsigned insn_data_size = 0; - if ( (pI->has_memory_read() || pI->has_memory_write()) ) { - if(!((inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))) - { + } + + addr_t insn_memaddr = 0xFEEBDAED; + memory_space_t insn_space = undefined_space; + _memory_op_t insn_memory_op = no_memory_op; + unsigned insn_data_size = 0; + if ((pI->has_memory_read() || pI->has_memory_write())) { + if (!((inst_opcode == MMA_LD_OP || inst_opcode == MMA_ST_OP))) { insn_memaddr = last_eaddr(); insn_space = last_space(); unsigned to_type = pI->get_type(); insn_data_size = datatype2size(to_type); insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; - } - } - - if ( pI->get_opcode() == BAR_OP && pI->barrier_op() == RED_OPTION) { - inst.add_callback( lane_id, last_callback().function, last_callback().instruction, this,false /*not atomic*/); - } + } + } - if ( pI->get_opcode() == ATOM_OP ) { + if (pI->get_opcode() == BAR_OP && pI->barrier_op() == RED_OPTION) { + inst.add_callback(lane_id, last_callback().function, + last_callback().instruction, this, + false /*not atomic*/); + } + + if (pI->get_opcode() == ATOM_OP) { insn_memaddr = last_eaddr(); insn_space = last_space(); - inst.add_callback( lane_id, last_callback().function, last_callback().instruction, this,true /*atomic*/); + inst.add_callback(lane_id, last_callback().function, + last_callback().instruction, this, true /*atomic*/); unsigned to_type = pI->get_type(); insn_data_size = datatype2size(to_type); - } + } - if (pI->get_opcode() == TEX_OP) { - inst.set_addr(lane_id, last_eaddr() ); - assert( inst.space == last_space() ); - insn_data_size = get_tex_datasize(pI, this); // texture obtain its data granularity from the texture info - } + if (pI->get_opcode() == TEX_OP) { + inst.set_addr(lane_id, last_eaddr()); + assert(inst.space == last_space()); + insn_data_size = get_tex_datasize( + pI, + this); // texture obtain its data granularity from the texture info + } - // Output register information to file and stdout - if( config.get_ptx_inst_debug_to_file()!=0 && - (config.get_ptx_inst_debug_thread_uid()==0||config.get_ptx_inst_debug_thread_uid()==get_uid()) ) { + // Output register information to file and stdout + if (config.get_ptx_inst_debug_to_file() != 0 && + (config.get_ptx_inst_debug_thread_uid() == 0 || + config.get_ptx_inst_debug_thread_uid() == get_uid())) { dump_modifiedregs(m_gpu->get_ptx_inst_debug_file()); dump_regs(m_gpu->get_ptx_inst_debug_file()); - } + } - if ( g_debug_execution >= 6 ) { - if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<6>(get_uid(), pc) ) - dump_modifiedregs(stdout); - } - if ( g_debug_execution >= 10 ) { - if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<10>(get_uid(), pc) ) - dump_regs(stdout); - } - update_pc(); - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn++; - - //not using it with functional simulation mode - if(!(this->m_functionalSimulationMode)) - ptx_file_line_stats_add_exec_count(pI); - - if ( m_gpu->gpgpu_ctx->func_sim->gpgpu_ptx_instruction_classification ) { + if (g_debug_execution >= 6) { + if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<6>(get_uid(), + pc)) + dump_modifiedregs(stdout); + } + if (g_debug_execution >= 10) { + if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<10>(get_uid(), + pc)) + dump_regs(stdout); + } + update_pc(); + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn++; + + // not using it with functional simulation mode + if (!(this->m_functionalSimulationMode)) + ptx_file_line_stats_add_exec_count(pI); + + if (m_gpu->gpgpu_ctx->func_sim->gpgpu_ptx_instruction_classification) { m_gpu->gpgpu_ctx->func_sim->init_inst_classification_stat(); - unsigned space_type=0; - switch ( pI->get_space().get_type() ) { - case global_space: space_type = 10; break; - case local_space: space_type = 11; break; - case tex_space: space_type = 12; break; - case surf_space: space_type = 13; break; - case param_space_kernel: - case param_space_local: - space_type = 14; break; - case shared_space: space_type = 15; break; - case const_space: space_type = 16; break; - default: - space_type = 0 ; - break; + unsigned space_type = 0; + switch (pI->get_space().get_type()) { + case global_space: + space_type = 10; + break; + case local_space: + space_type = 11; + break; + case tex_space: + space_type = 12; + break; + case surf_space: + space_type = 13; + break; + case param_space_kernel: + case param_space_local: + space_type = 14; + break; + case shared_space: + space_type = 15; + break; + case const_space: + space_type = 16; + break; + default: + space_type = 0; + break; } - StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], op_classification); - if (space_type) StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], ( int )space_type); - StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_op_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], (int) pI->get_opcode() ); - } - if ( (m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn % 100000) == 0 ) { + StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat + [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], + op_classification); + if (space_type) + StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat + [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], + (int)space_type); + StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_op_classification_stat + [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], + (int)pI->get_opcode()); + } + if ((m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn % 100000) == 0) { dim3 ctaid = get_ctaid(); dim3 tid = get_tid(); - DPRINTF(LIVENESS, "GPGPU-Sim PTX: %u instructions simulated : ctaid=(%u,%u,%u) tid=(%u,%u,%u)\n", - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, ctaid.x,ctaid.y,ctaid.z,tid.x,tid.y,tid.z ); + DPRINTF(LIVENESS, + "GPGPU-Sim PTX: %u instructions simulated : ctaid=(%u,%u,%u) " + "tid=(%u,%u,%u)\n", + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, ctaid.x, ctaid.y, + ctaid.z, tid.x, tid.y, tid.z); fflush(stdout); - } - - // "Return values" - if(!skip) { - if(!((inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))) - { - inst.space = insn_space; - inst.set_addr(lane_id, insn_memaddr); - inst.data_size = insn_data_size; // simpleAtomicIntrinsics - assert( inst.memory_op == insn_memory_op ); - } - } - - } catch ( int x ) { - printf("GPGPU-Sim PTX: ERROR (%d) executing intruction (%s:%u)\n", x, pI->source_file(), pI->source_line() ); - printf("GPGPU-Sim PTX: '%s'\n", pI->get_source() ); - abort(); - } - -} - -void cuda_sim::set_param_gpgpu_num_shaders(int num_shaders) -{ - gpgpu_param_num_shaders = num_shaders; -} - -const struct gpgpu_ptx_sim_info* ptx_sim_kernel_info(const function_info *kernel) -{ - return kernel->get_kernel_info(); -} - -const warp_inst_t *gpgpu_context::ptx_fetch_inst( address_type pc ) -{ - return pc_to_instruction(pc); -} - -unsigned ptx_sim_init_thread( kernel_info_t &kernel, - ptx_thread_info** thread_info, - int sid, - unsigned tid, - unsigned threads_left, - unsigned num_threads, - core_t *core, - unsigned hw_cta_id, - unsigned hw_warp_id, - gpgpu_t *gpu, - bool isInFunctionalSimulationMode) -{ - std::list &active_threads = kernel.active_threads(); - - static std::map shared_memory_lookup; - static std::map sstarr_memory_lookup; - static std::map ptx_cta_lookup; - static std::map ptx_warp_lookup; - static std::map > local_memory_lookup; - - if ( *thread_info != NULL ) { - ptx_thread_info *thd = *thread_info; - assert( thd->is_done() ); - if ( g_debug_execution==-1 ) { - dim3 ctaid = thd->get_ctaid(); - dim3 t = thd->get_tid(); - printf("GPGPU-Sim PTX simulator: thread exiting ctaid=(%u,%u,%u) tid=(%u,%u,%u) uid=%u\n", - ctaid.x,ctaid.y,ctaid.z,t.x,t.y,t.z, thd->get_uid() ); - fflush(stdout); - } - thd->m_cta_info->register_deleted_thread(thd); - delete thd; - *thread_info = NULL; - } - - if ( !active_threads.empty() ) { - assert( active_threads.size() <= threads_left ); - ptx_thread_info *thd = active_threads.front(); - active_threads.pop_front(); - *thread_info = thd; - thd->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, isInFunctionalSimulationMode ); - return 1; - } - - if ( kernel.no_more_ctas_to_run() ) { - return 0; //finished! - } - - if ( threads_left < kernel.threads_per_cta() ) { - return 0; - } - - if ( g_debug_execution==-1 ) { - printf("GPGPU-Sim PTX simulator: STARTING THREAD ALLOCATION --> \n"); - fflush(stdout); - } - - //initializing new CTA - ptx_cta_info *cta_info = NULL; - memory_space *shared_mem = NULL; - memory_space *sstarr_mem = NULL; - - unsigned cta_size = kernel.threads_per_cta(); - unsigned max_cta_per_sm = num_threads/cta_size; // e.g., 256 / 48 = 5 - assert( max_cta_per_sm > 0 ); - - //unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid; - unsigned sm_idx = hw_cta_id*gpu->gpgpu_ctx->func_sim->gpgpu_param_num_shaders + sid; - - if ( shared_memory_lookup.find(sm_idx) == shared_memory_lookup.end() ) { - if ( g_debug_execution >= 1 ) { - printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", - sm_idx, sid, max_cta_per_sm ); - } - char buf[512]; - snprintf(buf,512,"shared_%u", sid); - shared_mem = new memory_space_impl<16*1024>(buf,4); - shared_memory_lookup[sm_idx] = shared_mem; - snprintf(buf,512,"sstarr_%u", sid); - sstarr_mem = new memory_space_impl<16*1024>(buf,4); - sstarr_memory_lookup[sm_idx] = sstarr_mem; - cta_info = new ptx_cta_info(sm_idx, gpu->gpgpu_ctx); - ptx_cta_lookup[sm_idx] = cta_info; - } else { - if ( g_debug_execution >= 1 ) { - printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", - sm_idx, sid, max_cta_per_sm ); - } - shared_mem = shared_memory_lookup[sm_idx]; - sstarr_mem = sstarr_memory_lookup[sm_idx]; - cta_info = ptx_cta_lookup[sm_idx]; - cta_info->check_cta_thread_status_and_reset(); - } - - std::map &local_mem_lookup = local_memory_lookup[sid]; - while( kernel.more_threads_in_cta() ) { - dim3 ctaid3d = kernel.get_next_cta_id(); - unsigned new_tid = kernel.get_next_thread_id(); - dim3 tid3d = kernel.get_next_thread_id_3d(); - kernel.increment_thread_id(); - new_tid += tid; - ptx_thread_info *thd = new ptx_thread_info(kernel); - ptx_warp_info *warp_info = NULL; - if ( ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end() ) { - warp_info = new ptx_warp_info(); - ptx_warp_lookup[hw_warp_id] = warp_info; - } else { - warp_info = ptx_warp_lookup[hw_warp_id]; - } - thd->m_warp_info = warp_info; + } - memory_space *local_mem = NULL; - std::map::iterator l = local_mem_lookup.find(new_tid); - if ( l != local_mem_lookup.end() ) { - local_mem = l->second; - } else { - char buf[512]; - snprintf(buf,512,"local_%u_%u", sid, new_tid); - local_mem = new memory_space_impl<32>(buf,32); - local_mem_lookup[new_tid] = local_mem; - } - thd->set_info(kernel.entry()); - thd->set_nctaid(kernel.get_grid_dim()); - thd->set_ntid(kernel.get_cta_dim()); - thd->set_ctaid(ctaid3d); - thd->set_tid(tid3d); - if( kernel.entry()->get_ptx_version().extensions() ) - thd->cpy_tid_to_reg(tid3d); - thd->set_valid(); - thd->m_shared_mem = shared_mem; - thd->m_sstarr_mem = sstarr_mem; - function_info *finfo = thd->func_info(); - symbol_table *st = finfo->get_symtab(); - thd->func_info()->param_to_shared(thd->m_shared_mem,st); - thd->func_info()->param_to_shared(thd->m_sstarr_mem,st); - thd->m_cta_info = cta_info; - cta_info->add_thread(thd); - thd->m_local_mem = local_mem; - if ( g_debug_execution==-1 ) { - printf("GPGPU-Sim PTX simulator: allocating thread ctaid=(%u,%u,%u) tid=(%u,%u,%u) @ 0x%Lx\n", - ctaid3d.x,ctaid3d.y,ctaid3d.z,tid3d.x,tid3d.y,tid3d.z, (unsigned long long)thd ); - fflush(stdout); + // "Return values" + if (!skip) { + if (!((inst_opcode == MMA_LD_OP || inst_opcode == MMA_ST_OP))) { + inst.space = insn_space; + inst.set_addr(lane_id, insn_memaddr); + inst.data_size = insn_data_size; // simpleAtomicIntrinsics + assert(inst.memory_op == insn_memory_op); } - active_threads.push_back(thd); - } - if ( g_debug_execution==-1 ) { - printf("GPGPU-Sim PTX simulator: <-- FINISHING THREAD ALLOCATION\n"); - fflush(stdout); - } - - kernel.increment_cta_id(); + } - assert( active_threads.size() <= threads_left ); - *thread_info = active_threads.front(); - (*thread_info)->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid,isInFunctionalSimulationMode ); - active_threads.pop_front(); - return 1; + } catch (int x) { + printf("GPGPU-Sim PTX: ERROR (%d) executing intruction (%s:%u)\n", x, + pI->source_file(), pI->source_line()); + printf("GPGPU-Sim PTX: '%s'\n", pI->get_source()); + abort(); + } } -size_t get_kernel_code_size( class function_info *entry ) -{ - return entry->get_function_size(); +void cuda_sim::set_param_gpgpu_num_shaders(int num_shaders) { + gpgpu_param_num_shaders = num_shaders; } +const struct gpgpu_ptx_sim_info *ptx_sim_kernel_info( + const function_info *kernel) { + return kernel->get_kernel_info(); +} -kernel_info_t *cuda_sim::gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, - gpgpu_ptx_sim_arg_list_t args, - struct dim3 gridDim, - struct dim3 blockDim, - gpgpu_t *gpu ) -{ - kernel_info_t *result = new kernel_info_t(gridDim,blockDim,entry,gpu->getNameArrayMapping(),gpu->getNameInfoMapping()); - unsigned argcount=args.size(); - unsigned argn=1; - for( gpgpu_ptx_sim_arg_list_t::iterator a = args.begin(); a != args.end(); a++ ) { - entry->add_param_data(argcount-argn,&(*a)); - argn++; - } - entry->finalize(result->get_param_memory()); - g_ptx_kernel_count++; - fflush(stdout); - - return result; +const warp_inst_t *gpgpu_context::ptx_fetch_inst(address_type pc) { + return pc_to_instruction(pc); } -#include "../../version" -#include "detailed_version" +unsigned ptx_sim_init_thread(kernel_info_t &kernel, + ptx_thread_info **thread_info, int sid, + unsigned tid, unsigned threads_left, + unsigned num_threads, core_t *core, + unsigned hw_cta_id, unsigned hw_warp_id, + gpgpu_t *gpu, bool isInFunctionalSimulationMode) { + std::list &active_threads = kernel.active_threads(); -void print_splash() -{ - static int splash_printed=0; - if ( !splash_printed ) { - fprintf(stdout, "\n\n *** %s [build %s] ***\n\n\n", g_gpgpusim_version_string, g_gpgpusim_build_string ); - splash_printed=1; - } -} + static std::map shared_memory_lookup; + static std::map sstarr_memory_lookup; + static std::map ptx_cta_lookup; + static std::map ptx_warp_lookup; + static std::map > + local_memory_lookup; -void cuda_sim::gpgpu_ptx_sim_register_const_variable(void *hostVar, const char *deviceName, size_t size ) -{ - printf("GPGPU-Sim PTX registering constant %s (%zu bytes) to name mapping\n", deviceName, size ); - g_const_name_lookup[hostVar] = deviceName; -} + if (*thread_info != NULL) { + ptx_thread_info *thd = *thread_info; + assert(thd->is_done()); + if (g_debug_execution == -1) { + dim3 ctaid = thd->get_ctaid(); + dim3 t = thd->get_tid(); + printf( + "GPGPU-Sim PTX simulator: thread exiting ctaid=(%u,%u,%u) " + "tid=(%u,%u,%u) uid=%u\n", + ctaid.x, ctaid.y, ctaid.z, t.x, t.y, t.z, thd->get_uid()); + fflush(stdout); + } + thd->m_cta_info->register_deleted_thread(thd); + delete thd; + *thread_info = NULL; + } + + if (!active_threads.empty()) { + assert(active_threads.size() <= threads_left); + ptx_thread_info *thd = active_threads.front(); + active_threads.pop_front(); + *thread_info = thd; + thd->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, + isInFunctionalSimulationMode); + return 1; + } + + if (kernel.no_more_ctas_to_run()) { + return 0; // finished! + } + + if (threads_left < kernel.threads_per_cta()) { + return 0; + } + + if (g_debug_execution == -1) { + printf("GPGPU-Sim PTX simulator: STARTING THREAD ALLOCATION --> \n"); + fflush(stdout); + } + + // initializing new CTA + ptx_cta_info *cta_info = NULL; + memory_space *shared_mem = NULL; + memory_space *sstarr_mem = NULL; + + unsigned cta_size = kernel.threads_per_cta(); + unsigned max_cta_per_sm = num_threads / cta_size; // e.g., 256 / 48 = 5 + assert(max_cta_per_sm > 0); + + // unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid; + unsigned sm_idx = + hw_cta_id * gpu->gpgpu_ctx->func_sim->gpgpu_param_num_shaders + sid; + + if (shared_memory_lookup.find(sm_idx) == shared_memory_lookup.end()) { + if (g_debug_execution >= 1) { + printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", sm_idx, + sid, max_cta_per_sm); + } + char buf[512]; + snprintf(buf, 512, "shared_%u", sid); + shared_mem = new memory_space_impl<16 * 1024>(buf, 4); + shared_memory_lookup[sm_idx] = shared_mem; + snprintf(buf, 512, "sstarr_%u", sid); + sstarr_mem = new memory_space_impl<16 * 1024>(buf, 4); + sstarr_memory_lookup[sm_idx] = sstarr_mem; + cta_info = new ptx_cta_info(sm_idx, gpu->gpgpu_ctx); + ptx_cta_lookup[sm_idx] = cta_info; + } else { + if (g_debug_execution >= 1) { + printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", sm_idx, + sid, max_cta_per_sm); + } + shared_mem = shared_memory_lookup[sm_idx]; + sstarr_mem = sstarr_memory_lookup[sm_idx]; + cta_info = ptx_cta_lookup[sm_idx]; + cta_info->check_cta_thread_status_and_reset(); + } + + std::map &local_mem_lookup = + local_memory_lookup[sid]; + while (kernel.more_threads_in_cta()) { + dim3 ctaid3d = kernel.get_next_cta_id(); + unsigned new_tid = kernel.get_next_thread_id(); + dim3 tid3d = kernel.get_next_thread_id_3d(); + kernel.increment_thread_id(); + new_tid += tid; + ptx_thread_info *thd = new ptx_thread_info(kernel); + ptx_warp_info *warp_info = NULL; + if (ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end()) { + warp_info = new ptx_warp_info(); + ptx_warp_lookup[hw_warp_id] = warp_info; + } else { + warp_info = ptx_warp_lookup[hw_warp_id]; + } + thd->m_warp_info = warp_info; -void cuda_sim::gpgpu_ptx_sim_register_global_variable(void *hostVar, const char *deviceName, size_t size ) -{ - printf("GPGPU-Sim PTX registering global %s hostVar to name mapping\n", deviceName ); - g_global_name_lookup[hostVar] = deviceName; + memory_space *local_mem = NULL; + std::map::iterator l = + local_mem_lookup.find(new_tid); + if (l != local_mem_lookup.end()) { + local_mem = l->second; + } else { + char buf[512]; + snprintf(buf, 512, "local_%u_%u", sid, new_tid); + local_mem = new memory_space_impl<32>(buf, 32); + local_mem_lookup[new_tid] = local_mem; + } + thd->set_info(kernel.entry()); + thd->set_nctaid(kernel.get_grid_dim()); + thd->set_ntid(kernel.get_cta_dim()); + thd->set_ctaid(ctaid3d); + thd->set_tid(tid3d); + if (kernel.entry()->get_ptx_version().extensions()) + thd->cpy_tid_to_reg(tid3d); + thd->set_valid(); + thd->m_shared_mem = shared_mem; + thd->m_sstarr_mem = sstarr_mem; + function_info *finfo = thd->func_info(); + symbol_table *st = finfo->get_symtab(); + thd->func_info()->param_to_shared(thd->m_shared_mem, st); + thd->func_info()->param_to_shared(thd->m_sstarr_mem, st); + thd->m_cta_info = cta_info; + cta_info->add_thread(thd); + thd->m_local_mem = local_mem; + if (g_debug_execution == -1) { + printf( + "GPGPU-Sim PTX simulator: allocating thread ctaid=(%u,%u,%u) " + "tid=(%u,%u,%u) @ 0x%Lx\n", + ctaid3d.x, ctaid3d.y, ctaid3d.z, tid3d.x, tid3d.y, tid3d.z, + (unsigned long long)thd); + fflush(stdout); + } + active_threads.push_back(thd); + } + if (g_debug_execution == -1) { + printf("GPGPU-Sim PTX simulator: <-- FINISHING THREAD ALLOCATION\n"); + fflush(stdout); + } + + kernel.increment_cta_id(); + + assert(active_threads.size() <= threads_left); + *thread_info = active_threads.front(); + (*thread_info) + ->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, + isInFunctionalSimulationMode); + active_threads.pop_front(); + return 1; +} + +size_t get_kernel_code_size(class function_info *entry) { + return entry->get_function_size(); +} + +kernel_info_t *cuda_sim::gpgpu_opencl_ptx_sim_init_grid( + class function_info *entry, gpgpu_ptx_sim_arg_list_t args, + struct dim3 gridDim, struct dim3 blockDim, gpgpu_t *gpu) { + kernel_info_t *result = + new kernel_info_t(gridDim, blockDim, entry, gpu->getNameArrayMapping(), + gpu->getNameInfoMapping()); + unsigned argcount = args.size(); + unsigned argn = 1; + for (gpgpu_ptx_sim_arg_list_t::iterator a = args.begin(); a != args.end(); + a++) { + entry->add_param_data(argcount - argn, &(*a)); + argn++; + } + entry->finalize(result->get_param_memory()); + g_ptx_kernel_count++; + fflush(stdout); + + return result; } -void cuda_sim::gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to, gpgpu_t *gpu ) -{ - printf("GPGPU-Sim PTX: starting gpgpu_ptx_sim_memcpy_symbol with hostVar 0x%p\n", hostVar); - bool found_sym = false; - memory_space_t mem_region = undefined_space; - std::string sym_name; - - std::map::iterator c=gpu->gpgpu_ctx->func_sim->g_const_name_lookup.find(hostVar); - if ( c!=gpu->gpgpu_ctx->func_sim->g_const_name_lookup.end() ) { - found_sym = true; - sym_name = c->second; - mem_region = const_space; - } - std::map::iterator g=gpu->gpgpu_ctx->func_sim->g_global_name_lookup.find(hostVar); - if ( g!=gpu->gpgpu_ctx->func_sim->g_global_name_lookup.end() ) { - if ( found_sym ) { - printf("Execution error: PTX symbol \"%s\" w/ hostVar=0x%Lx is declared both const and global?\n", - sym_name.c_str(), (unsigned long long)hostVar ); - abort(); - } - found_sym = true; - sym_name = g->second; - mem_region = global_space; - } - if( g_globals.find(hostVar) != g_globals.end() ) { - found_sym = true; - sym_name = hostVar; - mem_region = global_space; - } - if( g_constants.find(hostVar) != g_constants.end() ) { - found_sym = true; - sym_name = hostVar; - mem_region = const_space; - } +#include "../../version" +#include "detailed_version" - if ( !found_sym ) { - printf("Execution error: No information for PTX symbol w/ hostVar=0x%Lx\n", (unsigned long long)hostVar ); +void print_splash() { + static int splash_printed = 0; + if (!splash_printed) { + fprintf(stdout, "\n\n *** %s [build %s] ***\n\n\n", + g_gpgpusim_version_string, g_gpgpusim_build_string); + splash_printed = 1; + } +} + +void cuda_sim::gpgpu_ptx_sim_register_const_variable(void *hostVar, + const char *deviceName, + size_t size) { + printf("GPGPU-Sim PTX registering constant %s (%zu bytes) to name mapping\n", + deviceName, size); + g_const_name_lookup[hostVar] = deviceName; +} + +void cuda_sim::gpgpu_ptx_sim_register_global_variable(void *hostVar, + const char *deviceName, + size_t size) { + printf("GPGPU-Sim PTX registering global %s hostVar to name mapping\n", + deviceName); + g_global_name_lookup[hostVar] = deviceName; +} + +void cuda_sim::gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, + size_t count, size_t offset, int to, + gpgpu_t *gpu) { + printf( + "GPGPU-Sim PTX: starting gpgpu_ptx_sim_memcpy_symbol with hostVar 0x%p\n", + hostVar); + bool found_sym = false; + memory_space_t mem_region = undefined_space; + std::string sym_name; + + std::map::iterator c = + gpu->gpgpu_ctx->func_sim->g_const_name_lookup.find(hostVar); + if (c != gpu->gpgpu_ctx->func_sim->g_const_name_lookup.end()) { + found_sym = true; + sym_name = c->second; + mem_region = const_space; + } + std::map::iterator g = + gpu->gpgpu_ctx->func_sim->g_global_name_lookup.find(hostVar); + if (g != gpu->gpgpu_ctx->func_sim->g_global_name_lookup.end()) { + if (found_sym) { + printf( + "Execution error: PTX symbol \"%s\" w/ hostVar=0x%Lx is declared " + "both const and global?\n", + sym_name.c_str(), (unsigned long long)hostVar); abort(); - } else printf("GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: Found PTX symbol w/ hostVar=0x%Lx\n", (unsigned long long)hostVar ); - const char *mem_name = NULL; - memory_space *mem = NULL; - - std::map::iterator st = gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.find(sym_name.c_str()); - assert( st != gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.end() ); - symbol_table *symtab = st->second; - - symbol *sym = symtab->lookup(sym_name.c_str()); - assert(sym); - unsigned dst = sym->get_address() + offset; - switch (mem_region.get_type()) { - case const_space: + } + found_sym = true; + sym_name = g->second; + mem_region = global_space; + } + if (g_globals.find(hostVar) != g_globals.end()) { + found_sym = true; + sym_name = hostVar; + mem_region = global_space; + } + if (g_constants.find(hostVar) != g_constants.end()) { + found_sym = true; + sym_name = hostVar; + mem_region = const_space; + } + + if (!found_sym) { + printf("Execution error: No information for PTX symbol w/ hostVar=0x%Lx\n", + (unsigned long long)hostVar); + abort(); + } else + printf( + "GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: Found PTX symbol w/ " + "hostVar=0x%Lx\n", + (unsigned long long)hostVar); + const char *mem_name = NULL; + memory_space *mem = NULL; + + std::map::iterator st = + gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.find(sym_name.c_str()); + assert(st != gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.end()); + symbol_table *symtab = st->second; + + symbol *sym = symtab->lookup(sym_name.c_str()); + assert(sym); + unsigned dst = sym->get_address() + offset; + switch (mem_region.get_type()) { + case const_space: mem = gpu->get_global_memory(); mem_name = "const"; break; - case global_space: + case global_space: mem = gpu->get_global_memory(); mem_name = "global"; break; - default: + default: abort(); - } - printf("GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: copying %s memory %zu bytes %s symbol %s+%zu @0x%x ...\n", - mem_name, count, (to?" to ":"from"), sym_name.c_str(), offset, dst ); - for ( unsigned n=0; n < count; n++ ) { - if( to ) mem->write(dst+n,1,((char*)src)+n,NULL,NULL); - else mem->read(dst+n,1,((char*)src)+n); - } - fflush(stdout); + } + printf( + "GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: copying %s memory %zu bytes " + "%s symbol %s+%zu @0x%x ...\n", + mem_name, count, (to ? " to " : "from"), sym_name.c_str(), offset, dst); + for (unsigned n = 0; n < count; n++) { + if (to) + mem->write(dst + n, 1, ((char *)src) + n, NULL, NULL); + else + mem->read(dst + n, 1, ((char *)src) + n); + } + fflush(stdout); } extern int ptx_debug; -void cuda_sim::read_sim_environment_variables() -{ - ptx_debug = 0; - g_debug_execution = 0; - g_interactive_debugger_enabled = false; - - char *mode = getenv("PTX_SIM_MODE_FUNC"); - if ( mode ) - sscanf(mode,"%u", &g_ptx_sim_mode); - printf("GPGPU-Sim PTX: simulation mode %d (can change with PTX_SIM_MODE_FUNC environment variable:\n", g_ptx_sim_mode); - printf(" 1=functional simulation only, 0=detailed performance simulator)\n"); - char *dbg_inter = getenv("GPGPUSIM_DEBUG"); - if ( dbg_inter && strlen(dbg_inter) ) { - printf("GPGPU-Sim PTX: enabling interactive debugger\n"); - fflush(stdout); - g_interactive_debugger_enabled = true; - } - char *dbg_level = getenv("PTX_SIM_DEBUG"); - if ( dbg_level && strlen(dbg_level) ) { - printf("GPGPU-Sim PTX: setting debug level to %s\n", dbg_level ); - fflush(stdout); - sscanf(dbg_level,"%d", &g_debug_execution); - } - char *dbg_thread = getenv("PTX_SIM_DEBUG_THREAD_UID"); - if ( dbg_thread && strlen(dbg_thread) ) { - printf("GPGPU-Sim PTX: printing debug information for thread uid %s\n", dbg_thread ); - fflush(stdout); - sscanf(dbg_thread,"%d", &g_debug_thread_uid); - } - char *dbg_pc = getenv("PTX_SIM_DEBUG_PC"); - if ( dbg_pc && strlen(dbg_pc) ) { - printf("GPGPU-Sim PTX: printing debug information for instruction with PC = %s\n", dbg_pc ); - fflush(stdout); - sscanf(dbg_pc,"%d", &g_debug_pc); - } +void cuda_sim::read_sim_environment_variables() { + ptx_debug = 0; + g_debug_execution = 0; + g_interactive_debugger_enabled = false; + + char *mode = getenv("PTX_SIM_MODE_FUNC"); + if (mode) sscanf(mode, "%u", &g_ptx_sim_mode); + printf( + "GPGPU-Sim PTX: simulation mode %d (can change with PTX_SIM_MODE_FUNC " + "environment variable:\n", + g_ptx_sim_mode); + printf( + " 1=functional simulation only, 0=detailed performance " + "simulator)\n"); + char *dbg_inter = getenv("GPGPUSIM_DEBUG"); + if (dbg_inter && strlen(dbg_inter)) { + printf("GPGPU-Sim PTX: enabling interactive debugger\n"); + fflush(stdout); + g_interactive_debugger_enabled = true; + } + char *dbg_level = getenv("PTX_SIM_DEBUG"); + if (dbg_level && strlen(dbg_level)) { + printf("GPGPU-Sim PTX: setting debug level to %s\n", dbg_level); + fflush(stdout); + sscanf(dbg_level, "%d", &g_debug_execution); + } + char *dbg_thread = getenv("PTX_SIM_DEBUG_THREAD_UID"); + if (dbg_thread && strlen(dbg_thread)) { + printf("GPGPU-Sim PTX: printing debug information for thread uid %s\n", + dbg_thread); + fflush(stdout); + sscanf(dbg_thread, "%d", &g_debug_thread_uid); + } + char *dbg_pc = getenv("PTX_SIM_DEBUG_PC"); + if (dbg_pc && strlen(dbg_pc)) { + printf( + "GPGPU-Sim PTX: printing debug information for instruction with PC = " + "%s\n", + dbg_pc); + fflush(stdout); + sscanf(dbg_pc, "%d", &g_debug_pc); + } #if CUDART_VERSION > 1010 - g_override_embedded_ptx = false; - char *usefile = getenv("PTX_SIM_USE_PTX_FILE"); - if (usefile && strlen(usefile)) { - printf("GPGPU-Sim PTX: overriding embedded ptx with ptx file (PTX_SIM_USE_PTX_FILE is set)\n"); - fflush(stdout); - g_override_embedded_ptx = true; - } - char *blocking = getenv("CUDA_LAUNCH_BLOCKING"); - if( blocking && !strcmp(blocking,"1") ) { - g_cuda_launch_blocking = true; - } + g_override_embedded_ptx = false; + char *usefile = getenv("PTX_SIM_USE_PTX_FILE"); + if (usefile && strlen(usefile)) { + printf( + "GPGPU-Sim PTX: overriding embedded ptx with ptx file " + "(PTX_SIM_USE_PTX_FILE is set)\n"); + fflush(stdout); + g_override_embedded_ptx = true; + } + char *blocking = getenv("CUDA_LAUNCH_BLOCKING"); + if (blocking && !strcmp(blocking, "1")) { + g_cuda_launch_blocking = true; + } #else - g_cuda_launch_blocking = true; - g_override_embedded_ptx = true; + g_cuda_launch_blocking = true; + g_override_embedded_ptx = true; #endif - if ( g_debug_execution >= 40 ) { - ptx_debug = 1; - } -} - -#define MAX(a,b) (((a)>(b))?(a):(b)) - -unsigned max_cta (const struct gpgpu_ptx_sim_info *kernel_info, unsigned threads_per_cta, unsigned int warp_size, unsigned int n_thread_per_shader, unsigned int gpgpu_shmem_size, unsigned int gpgpu_shader_registers, unsigned int max_cta_per_core) -{ - - unsigned int padded_cta_size = threads_per_cta; - if (padded_cta_size%warp_size) - padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size); - unsigned int result_thread = n_thread_per_shader / padded_cta_size; - - unsigned int result_shmem = (unsigned)-1; - if (kernel_info->smem > 0) - result_shmem = gpgpu_shmem_size / kernel_info->smem; - unsigned int result_regs = (unsigned)-1; - if (kernel_info->regs > 0) - result_regs = gpgpu_shader_registers / (padded_cta_size * ((kernel_info->regs+3)&~3)); - printf("padded cta size is %d and %d and %d",padded_cta_size, kernel_info->regs, ((kernel_info->regs+3)&~3) ); - //Limit by CTA - unsigned int result_cta = max_cta_per_core; - - unsigned result = result_thread; - result = gs_min2(result, result_shmem); - result = gs_min2(result, result_regs); - result = gs_min2(result, result_cta); - - printf ("GPGPU-Sim uArch: CTA/core = %u, limited by:", result); - if (result == result_thread) printf (" threads"); - if (result == result_shmem) printf (" shmem"); - if (result == result_regs) printf (" regs"); - if (result == result_cta) printf (" cta_limit"); - printf ("\n"); - - return result; + if (g_debug_execution >= 40) { + ptx_debug = 1; + } +} + +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) + +unsigned max_cta(const struct gpgpu_ptx_sim_info *kernel_info, + unsigned threads_per_cta, unsigned int warp_size, + unsigned int n_thread_per_shader, + unsigned int gpgpu_shmem_size, + unsigned int gpgpu_shader_registers, + unsigned int max_cta_per_core) { + unsigned int padded_cta_size = threads_per_cta; + if (padded_cta_size % warp_size) + padded_cta_size = ((padded_cta_size / warp_size) + 1) * (warp_size); + unsigned int result_thread = n_thread_per_shader / padded_cta_size; + + unsigned int result_shmem = (unsigned)-1; + if (kernel_info->smem > 0) + result_shmem = gpgpu_shmem_size / kernel_info->smem; + unsigned int result_regs = (unsigned)-1; + if (kernel_info->regs > 0) + result_regs = gpgpu_shader_registers / + (padded_cta_size * ((kernel_info->regs + 3) & ~3)); + printf("padded cta size is %d and %d and %d", padded_cta_size, + kernel_info->regs, ((kernel_info->regs + 3) & ~3)); + // Limit by CTA + unsigned int result_cta = max_cta_per_core; + + unsigned result = result_thread; + result = gs_min2(result, result_shmem); + result = gs_min2(result, result_regs); + result = gs_min2(result, result_cta); + + printf("GPGPU-Sim uArch: CTA/core = %u, limited by:", result); + if (result == result_thread) printf(" threads"); + if (result == result_shmem) printf(" shmem"); + if (result == result_regs) printf(" regs"); + if (result == result_cta) printf(" cta_limit"); + printf("\n"); + + return result; } /*! -This function simulates the CUDA code functionally, it takes a kernel_info_t parameter +This function simulates the CUDA code functionally, it takes a kernel_info_t +parameter which holds the data for the CUDA kernel to be executed !*/ -void cuda_sim::gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) -{ - printf("GPGPU-Sim: Performing Functional Simulation, executing kernel %s...\n",kernel.name().c_str()); - - //using a shader core object for book keeping, it is not needed but as most function built for performance simulation need it we use it here - //extern gpgpu_sim *g_the_gpu; - //before we execute, we should do PDOM analysis for functional simulation scenario. - function_info *kernel_func_info = kernel.entry(); - const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel_func_info); - checkpoint *g_checkpoint; - g_checkpoint = new checkpoint(); - - if (kernel_func_info->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", kernel.name().c_str() ); +void cuda_sim::gpgpu_cuda_ptx_sim_main_func(kernel_info_t &kernel, + bool openCL) { + printf( + "GPGPU-Sim: Performing Functional Simulation, executing kernel %s...\n", + kernel.name().c_str()); + + // using a shader core object for book keeping, it is not needed but as most + // function built for performance simulation need it we use it here + // extern gpgpu_sim *g_the_gpu; + // before we execute, we should do PDOM analysis for functional simulation + // scenario. + function_info *kernel_func_info = kernel.entry(); + const struct gpgpu_ptx_sim_info *kernel_info = + ptx_sim_kernel_info(kernel_func_info); + checkpoint *g_checkpoint; + g_checkpoint = new checkpoint(); + + if (kernel_func_info->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", + kernel.name().c_str()); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", + kernel.name().c_str()); + kernel_func_info->do_pdom(); + kernel_func_info->set_pdom(); + } + + unsigned max_cta_tot = max_cta( + kernel_info, kernel.threads_per_cta(), + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() + ->n_thread_per_shader, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() + ->gpgpu_shmem_size, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() + ->gpgpu_shader_registers, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() + ->max_cta_per_core); + printf("Max CTA : %d\n", max_cta_tot); + + int cp_op = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_option; + int cp_kernel = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_kernel; + cp_count = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_insn_Y; + cp_cta_resume = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_CTA_t; + int cta_launched = 0; + + // we excute the kernel one CTA (Block) at the time, as synchronization + // functions work block wise + while (!kernel.no_more_ctas_to_run()) { + unsigned temp = kernel.get_next_cta_id_single(); + + if (cp_op == 0 || (cp_op == 1 && cta_launched < cp_cta_resume && + kernel.get_uid() == cp_kernel) || + kernel.get_uid() < cp_kernel) // just fro testing + { + functionalCoreSim cta( + &kernel, gpgpu_ctx->the_gpgpusim->g_the_gpu, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size); + cta.execute(cp_count, temp); + +#if (CUDART_VERSION >= 5000) + gpgpu_ctx->device_runtime->launch_all_device_kernels(); +#endif } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", kernel.name().c_str() ); - kernel_func_info->do_pdom(); - kernel_func_info->set_pdom(); + kernel.increment_cta_id(); } - - unsigned max_cta_tot = max_cta(kernel_info,kernel.threads_per_cta(), gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->n_thread_per_shader, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->gpgpu_shmem_size, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->gpgpu_shader_registers, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->max_cta_per_core); - printf("Max CTA : %d\n",max_cta_tot); - - int cp_op= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_option; - int cp_kernel= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_kernel; - cp_count= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_insn_Y; - cp_cta_resume= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_CTA_t; - int cta_launched =0; - - //we excute the kernel one CTA (Block) at the time, as synchronization functions work block wise - while(!kernel.no_more_ctas_to_run()){ - unsigned temp=kernel.get_next_cta_id_single(); - - - if(cp_op==0 || (cp_op==1 && cta_launchedthe_gpgpusim->g_the_gpu, - gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size - ); - cta.execute(cp_count,temp); - - #if (CUDART_VERSION >= 5000) - gpgpu_ctx->device_runtime->launch_all_device_kernels(); - #endif - } - else - { - kernel.increment_cta_id(); - } cta_launched++; + } + + if (cp_op == 1) { + char f1name[2048]; + snprintf(f1name, 2048, "checkpoint_files/global_mem_%d.txt", + kernel.get_uid()); + g_checkpoint->store_global_mem( + gpgpu_ctx->the_gpgpusim->g_the_gpu->get_global_memory(), f1name, + (char *)"%08x"); + } + + // registering this kernel as done + + // openCL kernel simulation calls don't register the kernel so we don't + // register its exit + if (!openCL) { + // extern stream_manager *g_stream_manager; + gpgpu_ctx->the_gpgpusim->g_stream_manager->register_finished_kernel( + kernel.get_uid()); + } + + //******PRINTING******* + printf("GPGPU-Sim: Done functional simulation (%u instructions simulated).\n", + g_ptx_sim_num_insn); + if (gpgpu_ptx_instruction_classification) { + StatDisp(g_inst_classification_stat[g_ptx_kernel_count]); + StatDisp(g_inst_op_classification_stat[g_ptx_kernel_count]); + } + + // time_t variables used to calculate the total simulation time + // the start time of simulation is hold by the global variable + // g_simulation_starttime + // g_simulation_starttime is initilized by gpgpu_ptx_sim_init_perf() in + // gpgpusim_entrypoint.cc upon starting gpgpu-sim + time_t end_time, elapsed_time, days, hrs, minutes, sec; + end_time = time((time_t *)NULL); + elapsed_time = + MAX(end_time - gpgpu_ctx->the_gpgpusim->g_simulation_starttime, 1); + + // calculating and printing simulation time in terms of days, hours, minutes + // and seconds + days = elapsed_time / (3600 * 24); + hrs = elapsed_time / 3600 - 24 * days; + minutes = elapsed_time / 60 - 60 * (hrs + 24 * days); + sec = elapsed_time - 60 * (minutes + 60 * (hrs + 24 * days)); + + fflush(stderr); + printf( + "\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", + (unsigned)days, (unsigned)hrs, (unsigned)minutes, (unsigned)sec, + (unsigned)elapsed_time); + printf("gpgpu_simulation_rate = %u (inst/sec)\n", + (unsigned)(g_ptx_sim_num_insn / elapsed_time)); + fflush(stdout); +} + +void functionalCoreSim::initializeCTA(unsigned ctaid_cp) { + int ctaLiveThreads = 0; + symbol_table *symtab = m_kernel->entry()->get_symtab(); + + for (int i = 0; i < m_warp_count; i++) { + m_warpAtBarrier[i] = false; + m_liveThreadCount[i] = 0; + } + for (int i = 0; i < m_warp_count * m_warp_size; i++) m_thread[i] = NULL; + + // get threads for a cta + for (unsigned i = 0; i < m_kernel->threads_per_cta(); i++) { + ptx_sim_init_thread(*m_kernel, &m_thread[i], 0, i, + m_kernel->threads_per_cta() - i, + m_kernel->threads_per_cta(), this, 0, i / m_warp_size, + (gpgpu_t *)m_gpu, true); + assert(m_thread[i] != NULL && !m_thread[i]->is_done()); + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/thread_%d_0_reg.txt", i); + if (m_gpu->gpgpu_ctx->func_sim->cp_cta_resume == 1) + m_thread[i]->resume_reg_thread(fname, symtab); + ctaLiveThreads++; + } + + for (int k = 0; k < m_warp_count; k++) createWarp(k); +} + +void functionalCoreSim::createWarp(unsigned warpId) { + simt_mask_t initialMask; + unsigned liveThreadsCount = 0; + initialMask.set(); + for (int i = warpId * m_warp_size; i < warpId * m_warp_size + m_warp_size; + i++) { + if (m_thread[i] == NULL) + initialMask.reset(i - warpId * m_warp_size); + else + liveThreadsCount++; + } + + assert(m_thread[warpId * m_warp_size] != NULL); + m_simt_stack[warpId]->launch(m_thread[warpId * m_warp_size]->get_pc(), + initialMask); + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/warp_%d_0_simt.txt", warpId); + + if (m_gpu->gpgpu_ctx->func_sim->cp_cta_resume == 1) { + unsigned pc, rpc; + m_simt_stack[warpId]->resume(fname); + m_simt_stack[warpId]->get_pdom_stack_top_info(&pc, &rpc); + for (int i = warpId * m_warp_size; i < warpId * m_warp_size + m_warp_size; + i++) { + m_thread[i]->set_npc(pc); + m_thread[i]->update_pc(); + } + } + m_liveThreadCount[warpId] = liveThreadsCount; +} + +void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp) { + m_gpu->gpgpu_ctx->func_sim->cp_count = m_gpu->checkpoint_insn_Y; + m_gpu->gpgpu_ctx->func_sim->cp_cta_resume = m_gpu->checkpoint_CTA_t; + initializeCTA(ctaid_cp); + + int count = 0; + while (true) { + bool someOneLive = false; + bool allAtBarrier = true; + for (unsigned i = 0; i < m_warp_count; i++) { + executeWarp(i, allAtBarrier, someOneLive); + count++; } - - - if(cp_op==1) - { - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", kernel.get_uid() ); - g_checkpoint->store_global_mem(gpgpu_ctx->the_gpgpusim->g_the_gpu->get_global_memory(), f1name , (char *)"%08x"); - } - - - - - //registering this kernel as done - - //openCL kernel simulation calls don't register the kernel so we don't register its exit - if(!openCL) { - //extern stream_manager *g_stream_manager; - gpgpu_ctx->the_gpgpusim->g_stream_manager->register_finished_kernel(kernel.get_uid()); - } - - //******PRINTING******* - printf( "GPGPU-Sim: Done functional simulation (%u instructions simulated).\n", g_ptx_sim_num_insn ); - if ( gpgpu_ptx_instruction_classification ) { - StatDisp( g_inst_classification_stat[g_ptx_kernel_count]); - StatDisp ( g_inst_op_classification_stat[g_ptx_kernel_count]); - } - - //time_t variables used to calculate the total simulation time - //the start time of simulation is hold by the global variable g_simulation_starttime - //g_simulation_starttime is initilized by gpgpu_ptx_sim_init_perf() in gpgpusim_entrypoint.cc upon starting gpgpu-sim - time_t end_time, elapsed_time, days, hrs, minutes, sec; - end_time = time((time_t *)NULL); - elapsed_time = MAX(end_time - gpgpu_ctx->the_gpgpusim->g_simulation_starttime, 1); - - - //calculating and printing simulation time in terms of days, hours, minutes and seconds - days = elapsed_time/(3600*24); - hrs = elapsed_time/3600 - 24*days; - minutes = elapsed_time/60 - 60*(hrs + 24*days); - sec = elapsed_time - 60*(minutes + 60*(hrs + 24*days)); - - fflush(stderr); - printf("\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", - (unsigned)days, (unsigned)hrs, (unsigned)minutes, (unsigned)sec, (unsigned)elapsed_time ); - printf("gpgpu_simulation_rate = %u (inst/sec)\n", (unsigned)(g_ptx_sim_num_insn / elapsed_time) ); - fflush(stdout); -} - -void functionalCoreSim::initializeCTA(unsigned ctaid_cp) -{ - int ctaLiveThreads=0; - symbol_table * symtab= m_kernel->entry()->get_symtab(); - - for(int i=0; i< m_warp_count; i++){ - m_warpAtBarrier[i]=false; - m_liveThreadCount[i]=0; + if (inst_count > 0 && count > inst_count && + (m_kernel->get_uid() == m_gpu->checkpoint_kernel) && + (ctaid_cp >= m_gpu->checkpoint_CTA) && + (ctaid_cp < m_gpu->checkpoint_CTA_t) && m_gpu->checkpoint_option == 1) { + someOneLive = false; + break; } - for(int i=0; i< m_warp_count*m_warp_size;i++) - m_thread[i]=NULL; - - //get threads for a cta - for(unsigned i=0; ithreads_per_cta();i++) { - ptx_sim_init_thread(*m_kernel,&m_thread[i],0,i,m_kernel->threads_per_cta()-i,m_kernel->threads_per_cta(),this,0,i/m_warp_size,(gpgpu_t*)m_gpu, true); - assert(m_thread[i]!=NULL && !m_thread[i]->is_done()); - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/thread_%d_0_reg.txt",i ); - if(m_gpu->gpgpu_ctx->func_sim->cp_cta_resume==1) - m_thread[i]->resume_reg_thread(fname,symtab); - ctaLiveThreads++; + if (!someOneLive) break; + if (allAtBarrier) { + for (unsigned i = 0; i < m_warp_count; i++) m_warpAtBarrier[i] = false; } - - for(int k=0;klaunch(m_thread[warpId*m_warp_size]->get_pc(),initialMask); - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/warp_%d_0_simt.txt",warpId ); - - if(m_gpu->gpgpu_ctx->func_sim->cp_cta_resume==1) - { - unsigned pc,rpc; - m_simt_stack[warpId]->resume(fname); - m_simt_stack[warpId]->get_pdom_stack_top_info(&pc,&rpc); - for(int i=warpId*m_warp_size; iset_npc(pc); - m_thread[i]->update_pc(); - } - - } - m_liveThreadCount[warpId]= liveThreadsCount; -} - -void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp) - { - m_gpu->gpgpu_ctx->func_sim->cp_count= m_gpu->checkpoint_insn_Y; - m_gpu->gpgpu_ctx->func_sim->cp_cta_resume= m_gpu->checkpoint_CTA_t; - initializeCTA(ctaid_cp); - - int count=0; - while(true){ - bool someOneLive= false; - bool allAtBarrier = true; - for(unsigned i=0;i0 && count>inst_count && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cpcheckpoint_CTA_t) && m_gpu->checkpoint_option==1) - { - someOneLive=false; - break; - } - if(!someOneLive) break; - if(allAtBarrier){ - for(unsigned i=0;iget_next_cta_id_single(); + if (m_gpu->checkpoint_option == 1 && + (m_kernel->get_uid() == m_gpu->checkpoint_kernel) && + (ctaid_cp >= m_gpu->checkpoint_CTA) && + (ctaid_cp < m_gpu->checkpoint_CTA_t)) { + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/shared_mem_%d.txt", ctaid - 1); + g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname, + (char *)"%08x"); + for (int i = 0; i < 32 * m_warp_count; i++) { + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/thread_%d_%d_reg.txt", i, + ctaid - 1); + m_thread[i]->print_reg_thread(fname); + char f1name[2048]; + snprintf(f1name, 2048, "checkpoint_files/local_mem_thread_%d_%d_reg.txt", + i, ctaid - 1); + g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name, + (char *)"%08x"); + m_thread[i]->set_done(); + m_thread[i]->exitCore(); + m_thread[i]->registerExit(); } - checkpoint *g_checkpoint; - g_checkpoint = new checkpoint(); - - ptx_reg_t regval; - regval.u64= 123; - - unsigned ctaid =m_kernel->get_next_cta_id_single(); - if(m_gpu->checkpoint_option==1 && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cpcheckpoint_CTA_t)) - { - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/shared_mem_%d.txt",ctaid-1 ); - g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname , (char *)"%08x"); - for(int i=0; i<32*m_warp_count;i++) - { - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i,ctaid-1 ); - m_thread[i]->print_reg_thread(fname); - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i,ctaid-1 ); - g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name , (char *)"%08x"); - m_thread[i]->set_done(); - m_thread[i]->exitCore(); - m_thread[i]->registerExit(); - } - - for(int i=0;iprint_checkpoint(fp); - fclose(fp); - } - } - + for (int i = 0; i < m_warp_count; i++) { + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/warp_%d_%d_simt.txt", i, + ctaid - 1); + FILE *fp = fopen(fname, "w"); + assert(fp != NULL); + m_simt_stack[i]->print_checkpoint(fp); + fclose(fp); + } + } } -void functionalCoreSim::executeWarp(unsigned i, bool &allAtBarrier, bool & someOneLive) -{ - if(!m_warpAtBarrier[i] && m_liveThreadCount[i]!=0){ - warp_inst_t inst =getExecuteWarp(i); - execute_warp_inst_t(inst,i); - if(inst.isatomic()) inst.do_atomic(true); - if(inst.op==BARRIER_OP || inst.op==MEMORY_BARRIER_OP ) m_warpAtBarrier[i]=true; - updateSIMTStack( i, &inst ); - } - if(m_liveThreadCount[i]>0) someOneLive=true; - if(!m_warpAtBarrier[i]&& m_liveThreadCount[i]>0) allAtBarrier = false; +void functionalCoreSim::executeWarp(unsigned i, bool &allAtBarrier, + bool &someOneLive) { + if (!m_warpAtBarrier[i] && m_liveThreadCount[i] != 0) { + warp_inst_t inst = getExecuteWarp(i); + execute_warp_inst_t(inst, i); + if (inst.isatomic()) inst.do_atomic(true); + if (inst.op == BARRIER_OP || inst.op == MEMORY_BARRIER_OP) + m_warpAtBarrier[i] = true; + updateSIMTStack(i, &inst); + } + if (m_liveThreadCount[i] > 0) someOneLive = true; + if (!m_warpAtBarrier[i] && m_liveThreadCount[i] > 0) allAtBarrier = false; } -unsigned gpgpu_context::translate_pc_to_ptxlineno(unsigned pc) -{ - // this function assumes that the kernel fits inside a single PTX file - // function_info *pFunc = g_func_info; // assume that the current kernel is the one in query - const ptx_instruction *pInsn = pc_to_instruction(pc); - unsigned ptx_line_number = pInsn->source_line(); +unsigned gpgpu_context::translate_pc_to_ptxlineno(unsigned pc) { + // this function assumes that the kernel fits inside a single PTX file + // function_info *pFunc = g_func_info; // assume that the current kernel is + // the one in query + const ptx_instruction *pInsn = pc_to_instruction(pc); + unsigned ptx_line_number = pInsn->source_line(); - return ptx_line_number; + return ptx_line_number; } // ptxinfo parser -extern std::map get_duplicate(); +extern std::map get_duplicate(); static char *g_ptxinfo_kname = NULL; static struct gpgpu_ptx_sim_info g_ptxinfo; -static std::map g_duplicate; +static std::map g_duplicate; static const char *g_last_dup_type; -const char *get_ptxinfo_kname() -{ - return g_ptxinfo_kname; -} +const char *get_ptxinfo_kname() { return g_ptxinfo_kname; } -void print_ptxinfo() -{ - if(! get_ptxinfo_kname()){ - printf ("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", - g_ptxinfo.gmem, - g_ptxinfo.cmem); - } - if(get_ptxinfo_kname()){ - printf ("GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", - get_ptxinfo_kname(), - g_ptxinfo.regs, - g_ptxinfo.lmem, - g_ptxinfo.smem, - g_ptxinfo.cmem ); - } +void print_ptxinfo() { + if (!get_ptxinfo_kname()) { + printf("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", g_ptxinfo.gmem, + g_ptxinfo.cmem); + } + if (get_ptxinfo_kname()) { + printf( + "GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", + get_ptxinfo_kname(), g_ptxinfo.regs, g_ptxinfo.lmem, g_ptxinfo.smem, + g_ptxinfo.cmem); + } } - -struct gpgpu_ptx_sim_info get_ptxinfo() -{ - return g_ptxinfo; +struct gpgpu_ptx_sim_info get_ptxinfo() { + return g_ptxinfo; } -std::map get_duplicate() -{ - return g_duplicate; -} +std::map get_duplicate() { return g_duplicate; } -void ptxinfo_linenum( unsigned linenum ) -{ - g_duplicate[linenum] = g_last_dup_type; +void ptxinfo_linenum(unsigned linenum) { + g_duplicate[linenum] = g_last_dup_type; } -void ptxinfo_dup_type( const char *dup_type ) -{ - g_last_dup_type = dup_type; -} +void ptxinfo_dup_type(const char *dup_type) { g_last_dup_type = dup_type; } -void ptxinfo_function(const char *fname ) -{ - clear_ptxinfo(); - g_ptxinfo_kname = strdup(fname); +void ptxinfo_function(const char *fname) { + clear_ptxinfo(); + g_ptxinfo_kname = strdup(fname); } -void ptxinfo_regs( unsigned nregs ) -{ - g_ptxinfo.regs=nregs; -} +void ptxinfo_regs(unsigned nregs) { g_ptxinfo.regs = nregs; } -void ptxinfo_lmem( unsigned declared, unsigned system ) -{ - g_ptxinfo.lmem=declared+system; +void ptxinfo_lmem(unsigned declared, unsigned system) { + g_ptxinfo.lmem = declared + system; } -void ptxinfo_gmem( unsigned declared, unsigned system ) -{ - g_ptxinfo.gmem=declared+system; +void ptxinfo_gmem(unsigned declared, unsigned system) { + g_ptxinfo.gmem = declared + system; } -void ptxinfo_smem( unsigned declared, unsigned system ) -{ - g_ptxinfo.smem=declared+system; +void ptxinfo_smem(unsigned declared, unsigned system) { + g_ptxinfo.smem = declared + system; } -void ptxinfo_cmem( unsigned nbytes, unsigned bank ) -{ - g_ptxinfo.cmem+=nbytes; -} +void ptxinfo_cmem(unsigned nbytes, unsigned bank) { g_ptxinfo.cmem += nbytes; } -void clear_ptxinfo() -{ - free(g_ptxinfo_kname); - g_ptxinfo_kname=NULL; - g_ptxinfo.regs=0; - g_ptxinfo.lmem=0; - g_ptxinfo.smem=0; - g_ptxinfo.cmem=0; - g_ptxinfo.gmem=0; - g_ptxinfo.ptx_version=0; - g_ptxinfo.sm_target=0; +void clear_ptxinfo() { + free(g_ptxinfo_kname); + g_ptxinfo_kname = NULL; + g_ptxinfo.regs = 0; + g_ptxinfo.lmem = 0; + g_ptxinfo.smem = 0; + g_ptxinfo.cmem = 0; + g_ptxinfo.gmem = 0; + g_ptxinfo.ptx_version = 0; + g_ptxinfo.sm_target = 0; } +void ptxinfo_opencl_addinfo(std::map &kernels) { + if (!g_ptxinfo_kname) { + printf("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", g_ptxinfo.gmem, + g_ptxinfo.cmem); + clear_ptxinfo(); + return; + } -void ptxinfo_opencl_addinfo( std::map &kernels ) -{ - - if(! g_ptxinfo_kname) { - printf ("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", - g_ptxinfo.gmem, - g_ptxinfo.cmem); - clear_ptxinfo(); - return; - } - - if( !strcmp("__cuda_dummy_entry__",g_ptxinfo_kname) ) { - // this string produced by ptxas for empty ptx files (e.g., bandwidth test) - clear_ptxinfo(); - return; - } - std::map::iterator k=kernels.find(g_ptxinfo_kname); - if( k==kernels.end() ) { - printf ("GPGPU-Sim PTX: ERROR ** implementation for '%s' not found.\n", g_ptxinfo_kname ); - abort(); - } else { - printf ("GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", - g_ptxinfo_kname, - g_ptxinfo.regs, - g_ptxinfo.lmem, - g_ptxinfo.smem, - g_ptxinfo.cmem ); - function_info *finfo = k->second; - assert(finfo!=NULL); - finfo->set_kernel_info( g_ptxinfo ); - } - clear_ptxinfo(); -} - -struct rec_pts cuda_sim::find_reconvergence_points( function_info *finfo ) -{ - rec_pts tmp; - std::map::iterator r=g_rpts.find(finfo); - - if( r==g_rpts.end() ) { - int num_recon = finfo->get_num_reconvergence_pairs(); - - gpgpu_recon_t *kernel_recon_points = (struct gpgpu_recon_t*) calloc(num_recon, sizeof(struct gpgpu_recon_t)); - finfo->get_reconvergence_pairs(kernel_recon_points); - printf("GPGPU-Sim PTX: reconvergence points for %s...\n", finfo->get_name().c_str() ); - for (int i=0;iprint_insn(); - printf("\n"); - printf("GPGPU-Sim PTX: immediate post dominator @ " ); - if( kernel_recon_points[i].target_inst ) - kernel_recon_points[i].target_inst->print_insn(); - printf("\n"); - } - printf("GPGPU-Sim PTX: ... end of reconvergence points for %s\n", finfo->get_name().c_str() ); - - tmp.s_kernel_recon_points = kernel_recon_points; - tmp.s_num_recon = num_recon; - g_rpts[finfo] = tmp; - } else { - tmp = r->second; - } - return tmp; -} - -address_type get_return_pc( void *thd ) -{ - // function call return - ptx_thread_info *the_thread = (ptx_thread_info*)thd; - assert( the_thread != NULL ); - return the_thread->get_return_PC(); -} - -address_type cuda_sim::get_converge_point( address_type pc ) -{ - // the branch could encode the reconvergence point and/or a bit that indicates the - // reconvergence point is the return PC on the call stack in the case the branch has - // no immediate postdominator in the function (i.e., due to multiple return points). - - std::map::iterator f=g_pc_to_finfo.find(pc); - assert( f != g_pc_to_finfo.end() ); - function_info *finfo = f->second; - rec_pts tmp = find_reconvergence_points(finfo); - - int i=0; - for (; i < tmp.s_num_recon; ++i) { - if (tmp.s_kernel_recon_points[i].source_pc == pc) { - if( tmp.s_kernel_recon_points[i].target_pc == (unsigned) -2 ) { - return RECONVERGE_RETURN_PC; - } else { - return tmp.s_kernel_recon_points[i].target_pc; - } + if (!strcmp("__cuda_dummy_entry__", g_ptxinfo_kname)) { + // this string produced by ptxas for empty ptx files (e.g., bandwidth test) + clear_ptxinfo(); + return; + } + std::map::iterator k = + kernels.find(g_ptxinfo_kname); + if (k == kernels.end()) { + printf("GPGPU-Sim PTX: ERROR ** implementation for '%s' not found.\n", + g_ptxinfo_kname); + abort(); + } else { + printf( + "GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", + g_ptxinfo_kname, g_ptxinfo.regs, g_ptxinfo.lmem, g_ptxinfo.smem, + g_ptxinfo.cmem); + function_info *finfo = k->second; + assert(finfo != NULL); + finfo->set_kernel_info(g_ptxinfo); + } + clear_ptxinfo(); +} + +struct rec_pts cuda_sim::find_reconvergence_points(function_info *finfo) { + rec_pts tmp; + std::map::iterator r = g_rpts.find(finfo); + + if (r == g_rpts.end()) { + int num_recon = finfo->get_num_reconvergence_pairs(); + + gpgpu_recon_t *kernel_recon_points = + (struct gpgpu_recon_t *)calloc(num_recon, sizeof(struct gpgpu_recon_t)); + finfo->get_reconvergence_pairs(kernel_recon_points); + printf("GPGPU-Sim PTX: reconvergence points for %s...\n", + finfo->get_name().c_str()); + for (int i = 0; i < num_recon; i++) { + printf("GPGPU-Sim PTX: %2u (potential) branch divergence @ ", i + 1); + kernel_recon_points[i].source_inst->print_insn(); + printf("\n"); + printf("GPGPU-Sim PTX: immediate post dominator @ "); + if (kernel_recon_points[i].target_inst) + kernel_recon_points[i].target_inst->print_insn(); + printf("\n"); + } + printf("GPGPU-Sim PTX: ... end of reconvergence points for %s\n", + finfo->get_name().c_str()); + + tmp.s_kernel_recon_points = kernel_recon_points; + tmp.s_num_recon = num_recon; + g_rpts[finfo] = tmp; + } else { + tmp = r->second; + } + return tmp; +} + +address_type get_return_pc(void *thd) { + // function call return + ptx_thread_info *the_thread = (ptx_thread_info *)thd; + assert(the_thread != NULL); + return the_thread->get_return_PC(); +} + +address_type cuda_sim::get_converge_point(address_type pc) { + // the branch could encode the reconvergence point and/or a bit that indicates + // the + // reconvergence point is the return PC on the call stack in the case the + // branch has + // no immediate postdominator in the function (i.e., due to multiple return + // points). + + std::map::iterator f = g_pc_to_finfo.find(pc); + assert(f != g_pc_to_finfo.end()); + function_info *finfo = f->second; + rec_pts tmp = find_reconvergence_points(finfo); + + int i = 0; + for (; i < tmp.s_num_recon; ++i) { + if (tmp.s_kernel_recon_points[i].source_pc == pc) { + if (tmp.s_kernel_recon_points[i].target_pc == (unsigned)-2) { + return RECONVERGE_RETURN_PC; + } else { + return tmp.s_kernel_recon_points[i].target_pc; } - } - return NO_BRANCH_DIVERGENCE; + } + } + return NO_BRANCH_DIVERGENCE; } -void functionalCoreSim::warp_exit( unsigned warp_id ) -{ - for(int i=0;im_cta_info->register_deleted_thread(m_thread[i]); - delete m_thread[i]; - } +void functionalCoreSim::warp_exit(unsigned warp_id) { + for (int i = 0; i < m_warp_count * m_warp_size; i++) { + if (m_thread[i] != NULL) { + m_thread[i]->m_cta_info->register_deleted_thread(m_thread[i]); + delete m_thread[i]; } + } } diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h index 1be3d19..b828749 100644 --- a/src/cuda-sim/cuda-sim.h +++ b/src/cuda-sim/cuda-sim.h @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -28,13 +30,13 @@ #ifndef CUDASIM_H_INCLUDED #define CUDASIM_H_INCLUDED -#include "../abstract_hardware_model.h" -#include"../gpgpu-sim/shader.h" #include #include -#include #include -#include"ptx_sim.h" +#include +#include "../abstract_hardware_model.h" +#include "../gpgpu-sim/shader.h" +#include "ptx_sim.h" class gpgpu_context; class memory_space; @@ -44,69 +46,65 @@ class symbol_table; extern const char *g_gpgpusim_version_string; extern int g_debug_execution; -extern void print_splash(); +extern void print_splash(); -extern void ptxinfo_opencl_addinfo( std::map &kernels ); -unsigned ptx_sim_init_thread( kernel_info_t &kernel, - class ptx_thread_info** thread_info, - int sid, - unsigned tid, - unsigned threads_left, - unsigned num_threads, - class core_t *core, - unsigned hw_cta_id, - unsigned hw_warp_id, - gpgpu_t *gpu, - bool functionalSimulationMode = false); -const struct gpgpu_ptx_sim_info* ptx_sim_kernel_info(const class function_info *kernel); +extern void ptxinfo_opencl_addinfo( + std::map &kernels); +unsigned ptx_sim_init_thread(kernel_info_t &kernel, + class ptx_thread_info **thread_info, int sid, + unsigned tid, unsigned threads_left, + unsigned num_threads, class core_t *core, + unsigned hw_cta_id, unsigned hw_warp_id, + gpgpu_t *gpu, + bool functionalSimulationMode = false); +const struct gpgpu_ptx_sim_info *ptx_sim_kernel_info( + const class function_info *kernel); /*! - * This class functionally executes a kernel. It uses the basic data structures and procedures in core_t + * This class functionally executes a kernel. It uses the basic data structures + * and procedures in core_t */ -class functionalCoreSim: public core_t -{ -public: - functionalCoreSim(kernel_info_t * kernel, gpgpu_sim *g, unsigned warp_size) - : core_t( g, kernel, warp_size, kernel->threads_per_cta() ) - { - m_warpAtBarrier = new bool [m_warp_count]; - m_liveThreadCount = new unsigned [m_warp_count]; - } - virtual ~functionalCoreSim(){ - warp_exit(0); - delete[] m_liveThreadCount; - delete[] m_warpAtBarrier; - } - //! executes all warps till completion - void execute(int inst_count, unsigned ctaid_cp); - virtual void warp_exit( unsigned warp_id ); - virtual bool warp_waiting_at_barrier( unsigned warp_id ) const - { - return (m_warpAtBarrier[warp_id] || !(m_liveThreadCount[warp_id]>0)); - } - -private: - void executeWarp(unsigned, bool &, bool &); - //initializes threads in the CTA block which we are executing - void initializeCTA(unsigned ctaid_cp); - virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid) - { - if(m_thread[tid]==NULL || m_thread[tid]->is_done()){ - m_liveThreadCount[tid/m_warp_size]--; - } +class functionalCoreSim : public core_t { + public: + functionalCoreSim(kernel_info_t *kernel, gpgpu_sim *g, unsigned warp_size) + : core_t(g, kernel, warp_size, kernel->threads_per_cta()) { + m_warpAtBarrier = new bool[m_warp_count]; + m_liveThreadCount = new unsigned[m_warp_count]; + } + virtual ~functionalCoreSim() { + warp_exit(0); + delete[] m_liveThreadCount; + delete[] m_warpAtBarrier; + } + //! executes all warps till completion + void execute(int inst_count, unsigned ctaid_cp); + virtual void warp_exit(unsigned warp_id); + virtual bool warp_waiting_at_barrier(unsigned warp_id) const { + return (m_warpAtBarrier[warp_id] || !(m_liveThreadCount[warp_id] > 0)); + } + + private: + void executeWarp(unsigned, bool &, bool &); + // initializes threads in the CTA block which we are executing + void initializeCTA(unsigned ctaid_cp); + virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, + unsigned tid) { + if (m_thread[tid] == NULL || m_thread[tid]->is_done()) { + m_liveThreadCount[tid / m_warp_size]--; } - - // lunches the stack and set the threads count - void createWarp(unsigned warpId); - - //each warp live thread count and barrier indicator - unsigned * m_liveThreadCount; - bool* m_warpAtBarrier; + } + + // lunches the stack and set the threads count + void createWarp(unsigned warpId); + + // each warp live thread count and barrier indicator + unsigned *m_liveThreadCount; + bool *m_warpAtBarrier; }; #define RECONVERGE_RETURN_PC ((address_type)-2) #define NO_BRANCH_DIVERGENCE ((address_type)-1) -address_type get_return_pc( void *thd ); +address_type get_return_pc(void *thd); const char *get_ptxinfo_kname(); void print_ptxinfo(); void clear_ptxinfo(); @@ -114,89 +112,98 @@ struct gpgpu_ptx_sim_info get_ptxinfo(); class gpgpu_recon_t; struct rec_pts { - gpgpu_recon_t *s_kernel_recon_points; - int s_num_recon; + gpgpu_recon_t *s_kernel_recon_points; + int s_num_recon; }; - class cuda_sim { - public: - cuda_sim( gpgpu_context* ctx ) { - g_ptx_sim_num_insn = 0; - g_ptx_kernel_count = -1; // used for classification stat collection purposes - gpgpu_param_num_shaders = 0; - g_cuda_launch_blocking = false; - g_inst_classification_stat = NULL; - g_inst_op_classification_stat= NULL; - g_assemble_code_next_pc=0; - g_debug_thread_uid = 0; - g_override_embedded_ptx = false; - ptx_tex_regs = NULL; - g_ptx_thread_info_delete_count=0; - g_ptx_thread_info_uid_next=1; - g_debug_pc = 0xBEEF1518; - gpgpu_ctx = ctx; - } - //global variables - char *opcode_latency_int; - char *opcode_latency_fp; - char *opcode_latency_dp; - char *opcode_latency_sfu; - char *opcode_latency_tensor; - char *opcode_initiation_int; - char *opcode_initiation_fp; - char *opcode_initiation_dp; - char *opcode_initiation_sfu; - char *opcode_initiation_tensor; - int cp_count; - int cp_cta_resume; - int g_ptxinfo_error_detected; - unsigned g_ptx_sim_num_insn; - char *cdp_latency_str; - int g_ptx_kernel_count; // used for classification stat collection purposes - std::map g_global_name_lookup; // indexed by hostVar - std::map g_const_name_lookup; // indexed by hostVar - int g_ptx_sim_mode; // if non-zero run functional simulation only (i.e., no notion of a clock cycle) - unsigned gpgpu_param_num_shaders; - class std::map g_rpts; - bool g_cuda_launch_blocking; - void ** g_inst_classification_stat; - void ** g_inst_op_classification_stat; - std::set g_globals; - std::set g_constants; - std::map g_pc_to_finfo; - int gpgpu_ptx_instruction_classification; - unsigned cdp_latency[5]; - unsigned g_assemble_code_next_pc; - int g_debug_thread_uid; - bool g_override_embedded_ptx; - std::set g_ptx_cta_info_sm_idx_used; - ptx_reg_t* ptx_tex_regs; - unsigned g_ptx_thread_info_delete_count; - unsigned g_ptx_thread_info_uid_next; - addr_t g_debug_pc; - // backward pointer - class gpgpu_context* gpgpu_ctx; - //global functions - void ptx_opcocde_latency_options (option_parser_t opp); - void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL = false ); - int gpgpu_opencl_ptx_sim_main_func( kernel_info_t *grid ); - void init_inst_classification_stat(); - kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, - gpgpu_ptx_sim_arg_list_t args, - struct dim3 gridDim, - struct dim3 blockDim, - gpgpu_t *gpu ); - void gpgpu_ptx_sim_register_global_variable(void *hostVar, const char *deviceName, size_t size ); - void gpgpu_ptx_sim_register_const_variable(void*, const char *deviceName, size_t size ); - void read_sim_environment_variables(); - void set_param_gpgpu_num_shaders(int num_shaders); - struct rec_pts find_reconvergence_points( function_info *finfo ); - address_type get_converge_point( address_type pc ); - void gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to, gpgpu_t *gpu ); - void ptx_print_insn( address_type pc, FILE *fp ); - std::string ptx_get_insn_str( address_type pc ); - template bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc); + public: + cuda_sim(gpgpu_context *ctx) { + g_ptx_sim_num_insn = 0; + g_ptx_kernel_count = + -1; // used for classification stat collection purposes + gpgpu_param_num_shaders = 0; + g_cuda_launch_blocking = false; + g_inst_classification_stat = NULL; + g_inst_op_classification_stat = NULL; + g_assemble_code_next_pc = 0; + g_debug_thread_uid = 0; + g_override_embedded_ptx = false; + ptx_tex_regs = NULL; + g_ptx_thread_info_delete_count = 0; + g_ptx_thread_info_uid_next = 1; + g_debug_pc = 0xBEEF1518; + gpgpu_ctx = ctx; + } + // global variables + char *opcode_latency_int; + char *opcode_latency_fp; + char *opcode_latency_dp; + char *opcode_latency_sfu; + char *opcode_latency_tensor; + char *opcode_initiation_int; + char *opcode_initiation_fp; + char *opcode_initiation_dp; + char *opcode_initiation_sfu; + char *opcode_initiation_tensor; + int cp_count; + int cp_cta_resume; + int g_ptxinfo_error_detected; + unsigned g_ptx_sim_num_insn; + char *cdp_latency_str; + int g_ptx_kernel_count; // used for classification stat collection purposes + std::map + g_global_name_lookup; // indexed by hostVar + std::map + g_const_name_lookup; // indexed by hostVar + int g_ptx_sim_mode; // if non-zero run functional simulation only (i.e., no + // notion of a clock cycle) + unsigned gpgpu_param_num_shaders; + class std::map g_rpts; + bool g_cuda_launch_blocking; + void **g_inst_classification_stat; + void **g_inst_op_classification_stat; + std::set g_globals; + std::set g_constants; + std::map g_pc_to_finfo; + int gpgpu_ptx_instruction_classification; + unsigned cdp_latency[5]; + unsigned g_assemble_code_next_pc; + int g_debug_thread_uid; + bool g_override_embedded_ptx; + std::set g_ptx_cta_info_sm_idx_used; + ptx_reg_t *ptx_tex_regs; + unsigned g_ptx_thread_info_delete_count; + unsigned g_ptx_thread_info_uid_next; + addr_t g_debug_pc; + // backward pointer + class gpgpu_context *gpgpu_ctx; + // global functions + void ptx_opcocde_latency_options(option_parser_t opp); + void gpgpu_cuda_ptx_sim_main_func(kernel_info_t &kernel, bool openCL = false); + int gpgpu_opencl_ptx_sim_main_func(kernel_info_t *grid); + void init_inst_classification_stat(); + kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, + gpgpu_ptx_sim_arg_list_t args, + struct dim3 gridDim, + struct dim3 blockDim, + gpgpu_t *gpu); + void gpgpu_ptx_sim_register_global_variable(void *hostVar, + const char *deviceName, + size_t size); + void gpgpu_ptx_sim_register_const_variable(void *, const char *deviceName, + size_t size); + void read_sim_environment_variables(); + void set_param_gpgpu_num_shaders(int num_shaders); + struct rec_pts find_reconvergence_points(function_info *finfo); + address_type get_converge_point(address_type pc); + void gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, + size_t count, size_t offset, int to, + gpgpu_t *gpu); + void ptx_print_insn(address_type pc, FILE *fp); + std::string ptx_get_insn_str(address_type pc); + template + bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc); }; #endif diff --git a/src/cuda-sim/cuda_device_printf.cc b/src/cuda-sim/cuda_device_printf.cc index 9ac0727..5088a88 100644 --- a/src/cuda-sim/cuda_device_printf.cc +++ b/src/cuda-sim/cuda_device_printf.cc @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -28,87 +30,90 @@ #include "cuda_device_printf.h" #include "ptx_ir.h" -void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand_info &op, memory_space *&mem, addr_t &addr); +void decode_space(memory_space_t &space, ptx_thread_info *thread, + const operand_info &op, memory_space *&mem, addr_t &addr); -void my_cuda_printf(const char *fmtstr,const char *arg_list) -{ - FILE *fp = stdout; - unsigned i=0,j=0; - unsigned arg_offset=0; - char buf[64]; - bool in_fmt=false; - while( fmtstr[i] ) { - char c = fmtstr[i++]; - if( !in_fmt ) { - if( c != '%' ) { - fprintf(fp,"%c",c); - } else { - in_fmt=true; - buf[0] = c; - j=1; - } +void my_cuda_printf(const char *fmtstr, const char *arg_list) { + FILE *fp = stdout; + unsigned i = 0, j = 0; + unsigned arg_offset = 0; + char buf[64]; + bool in_fmt = false; + while (fmtstr[i]) { + char c = fmtstr[i++]; + if (!in_fmt) { + if (c != '%') { + fprintf(fp, "%c", c); } else { - if(!( c == 'u' || c == 'f' || c == 'd' )) { - printf("GPGPU-Sim PTX: ERROR ** printf parsing support is limited to %%u, %%f, %%d at present"); - abort(); - } - buf[j] = c; - buf[j+1] = 0; - void* ptr = (void*)&arg_list[arg_offset]; - //unsigned long long value = ((unsigned long long*)arg_list)[arg_offset]; - if( c == 'u' || c == 'd' ) { - fprintf(fp,buf,*((unsigned long long*)ptr)); - } else if( c == 'f' ) { - double tmp = *((double*)ptr); - fprintf(fp,buf,tmp); - } - arg_offset++; - in_fmt=false; + in_fmt = true; + buf[0] = c; + j = 1; } - } + } else { + if (!(c == 'u' || c == 'f' || c == 'd')) { + printf( + "GPGPU-Sim PTX: ERROR ** printf parsing support is limited to %%u, " + "%%f, %%d at present"); + abort(); + } + buf[j] = c; + buf[j + 1] = 0; + void *ptr = (void *)&arg_list[arg_offset]; + // unsigned long long value = ((unsigned long long*)arg_list)[arg_offset]; + if (c == 'u' || c == 'd') { + fprintf(fp, buf, *((unsigned long long *)ptr)); + } else if (c == 'f') { + double tmp = *((double *)ptr); + fprintf(fp, buf, tmp); + } + arg_offset++; + in_fmt = false; + } + } } -void gpgpusim_cuda_vprintf(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func ) -{ - char *fmtstr = NULL; - char *arg_list = NULL; - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - assert( n_args == 2 ); - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); - const symbol *formal_param = target_func->get_arg(arg); - unsigned size=formal_param->get_size_in_bytes(); - assert( formal_param->is_param_local() ); - assert( actual_param_op.is_param_local() ); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - unsigned long long buffer[1024]; - assert(size<1024*sizeof(unsigned long long)); - thread->m_local_mem->read(from_addr,size,buffer); - addr_t addr = (addr_t)buffer[0]; // should be pointer to generic memory location - memory_space *mem=NULL; - memory_space_t space = generic_space; - decode_space(space,thread,actual_param_op,mem,addr); // figure out which space - if( arg == 0 ) { - unsigned len = 0; - char b = 0; - do { // figure out length - mem->read(addr+len,1,&b); - len++; - } while(b); - fmtstr = (char*)malloc(len+64); - for( int i=0; i < len; i++ ) - mem->read(addr+i,1,fmtstr+i); - //mem->read(addr,len,fmtstr); - } else { - unsigned len = thread->get_finfo()->local_mem_framesize(); - arg_list = (char*)malloc(len+64); - for( int i=0; i < len; i++ ) - mem->read(addr+i,1,arg_list+i); - //mem->read(addr,len,arg_list); - } - } - my_cuda_printf(fmtstr,arg_list); - free(fmtstr); - free(arg_list); +void gpgpusim_cuda_vprintf(const ptx_instruction *pI, ptx_thread_info *thread, + const function_info *target_func) { + char *fmtstr = NULL; + char *arg_list = NULL; + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + assert(n_args == 2); + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); + const symbol *formal_param = target_func->get_arg(arg); + unsigned size = formal_param->get_size_in_bytes(); + assert(formal_param->is_param_local()); + assert(actual_param_op.is_param_local()); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + unsigned long long buffer[1024]; + assert(size < 1024 * sizeof(unsigned long long)); + thread->m_local_mem->read(from_addr, size, buffer); + addr_t addr = + (addr_t)buffer[0]; // should be pointer to generic memory location + memory_space *mem = NULL; + memory_space_t space = generic_space; + decode_space(space, thread, actual_param_op, mem, + addr); // figure out which space + if (arg == 0) { + unsigned len = 0; + char b = 0; + do { // figure out length + mem->read(addr + len, 1, &b); + len++; + } while (b); + fmtstr = (char *)malloc(len + 64); + for (int i = 0; i < len; i++) mem->read(addr + i, 1, fmtstr + i); + // mem->read(addr,len,fmtstr); + } else { + unsigned len = thread->get_finfo()->local_mem_framesize(); + arg_list = (char *)malloc(len + 64); + for (int i = 0; i < len; i++) mem->read(addr + i, 1, arg_list + i); + // mem->read(addr,len,arg_list); + } + } + my_cuda_printf(fmtstr, arg_list); + free(fmtstr); + free(arg_list); } diff --git a/src/cuda-sim/cuda_device_printf.h b/src/cuda-sim/cuda_device_printf.h index 4e9baaa..20744bb 100644 --- a/src/cuda-sim/cuda_device_printf.h +++ b/src/cuda-sim/cuda_device_printf.h @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -28,6 +30,8 @@ #ifndef CUDA_DEVICE_PRINTF_INCLUDED #define CUDA_DEVICE_PRINTF_INCLUDED -void gpgpusim_cuda_vprintf(const class ptx_instruction * pI, class ptx_thread_info * thread, const class function_info * target_func ); +void gpgpusim_cuda_vprintf(const class ptx_instruction* pI, + class ptx_thread_info* thread, + const class function_info* target_func); #endif diff --git a/src/cuda-sim/cuda_device_runtime.cc b/src/cuda-sim/cuda_device_runtime.cc index 4baced5..257dd50 100644 --- a/src/cuda-sim/cuda_device_runtime.cc +++ b/src/cuda-sim/cuda_device_runtime.cc @@ -1,297 +1,329 @@ -//Jin: cuda_device_runtime.cc -//Defines CUDA device runtime APIs for CDP support - +// Jin: cuda_device_runtime.cc +// Defines CUDA device runtime APIs for CDP support #include #include - - #if (CUDART_VERSION >= 5000) #define __CUDA_RUNTIME_API_H__ #include #include +#include "../../libcuda/gpgpu_context.h" #include "../gpgpu-sim/gpu-sim.h" -#include "cuda-sim.h" -#include "ptx_ir.h" -#include "../stream_manager.h" #include "../gpgpusim_entrypoint.h" +#include "../stream_manager.h" +#include "cuda-sim.h" #include "cuda_device_runtime.h" -#include "../../libcuda/gpgpu_context.h" +#include "ptx_ir.h" -#define DEV_RUNTIME_REPORT(a) \ - if( g_debug_execution ) { \ - std::cout << __FILE__ << ", " << __LINE__ << ": " << a << "\n"; \ - std::cout.flush(); \ - } - - - -//Handling device runtime api: -//void * cudaGetParameterBufferV2(void *func, dim3 gridDimension, dim3 blockDimension, unsigned int sharedMemSize) -void cuda_device_runtime::gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) -{ - DEV_RUNTIME_REPORT("Calling cudaGetParameterBufferV2"); - - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert( n_args == 4 ); - - function_info * child_kernel_entry; - struct dim3 grid_dim, block_dim; - unsigned int shared_mem; - - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# - const symbol *formal_param = target_func->get_arg(arg); //cudaGetParameterBufferV2_param_# - unsigned size=formal_param->get_size_in_bytes(); - assert( formal_param->is_param_local() ); - assert( actual_param_op.is_param_local() ); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - - if(arg == 0) {//function_info* for the child kernel - unsigned long long buf; - assert(size == sizeof(function_info *)); - thread->m_local_mem->read(from_addr, size, &buf); - child_kernel_entry = (function_info *)buf; - assert(child_kernel_entry); - DEV_RUNTIME_REPORT("child kernel name " << child_kernel_entry->get_name()); - } - else if(arg == 1) { //dim3 grid_dim for the child kernel - assert(size == sizeof(struct dim3)); - thread->m_local_mem->read(from_addr, size, & grid_dim); - DEV_RUNTIME_REPORT("grid (" << grid_dim.x << ", " << grid_dim.y << ", " << grid_dim.z << ")"); - } - else if(arg == 2) { //dim3 block_dim for the child kernel - assert(size == sizeof(struct dim3)); - thread->m_local_mem->read(from_addr, size, & block_dim); - DEV_RUNTIME_REPORT("block (" << block_dim.x << ", " << block_dim.y << ", " << block_dim.z << ")"); - } - else if(arg == 3) { //unsigned int shared_mem - assert(size == sizeof(unsigned int)); - thread->m_local_mem->read(from_addr, size, & shared_mem); - DEV_RUNTIME_REPORT("shared memory " << shared_mem); - } +#define DEV_RUNTIME_REPORT(a) \ + if (g_debug_execution) { \ + std::cout << __FILE__ << ", " << __LINE__ << ": " << a << "\n"; \ + std::cout.flush(); \ + } + +// Handling device runtime api: +// void * cudaGetParameterBufferV2(void *func, dim3 gridDimension, dim3 +// blockDimension, unsigned int sharedMemSize) +void cuda_device_runtime::gpgpusim_cuda_getParameterBufferV2( + const ptx_instruction *pI, ptx_thread_info *thread, + const function_info *target_func) { + DEV_RUNTIME_REPORT("Calling cudaGetParameterBufferV2"); + + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert(n_args == 4); + + function_info *child_kernel_entry; + struct dim3 grid_dim, block_dim; + unsigned int shared_mem; + + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); // param# + const symbol *formal_param = + target_func->get_arg(arg); // cudaGetParameterBufferV2_param_# + unsigned size = formal_param->get_size_in_bytes(); + assert(formal_param->is_param_local()); + assert(actual_param_op.is_param_local()); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + + if (arg == 0) { // function_info* for the child kernel + unsigned long long buf; + assert(size == sizeof(function_info *)); + thread->m_local_mem->read(from_addr, size, &buf); + child_kernel_entry = (function_info *)buf; + assert(child_kernel_entry); + DEV_RUNTIME_REPORT("child kernel name " + << child_kernel_entry->get_name()); + } else if (arg == 1) { // dim3 grid_dim for the child kernel + assert(size == sizeof(struct dim3)); + thread->m_local_mem->read(from_addr, size, &grid_dim); + DEV_RUNTIME_REPORT("grid (" << grid_dim.x << ", " << grid_dim.y << ", " + << grid_dim.z << ")"); + } else if (arg == 2) { // dim3 block_dim for the child kernel + assert(size == sizeof(struct dim3)); + thread->m_local_mem->read(from_addr, size, &block_dim); + DEV_RUNTIME_REPORT("block (" << block_dim.x << ", " << block_dim.y << ", " + << block_dim.z << ")"); + } else if (arg == 3) { // unsigned int shared_mem + assert(size == sizeof(unsigned int)); + thread->m_local_mem->read(from_addr, size, &shared_mem); + DEV_RUNTIME_REPORT("shared memory " << shared_mem); } - - //get total child kernel argument size and malloc buffer in global memory - unsigned child_kernel_arg_size = child_kernel_entry->get_args_aligned_size(); - void * param_buffer = thread->get_gpu()->gpu_malloc(child_kernel_arg_size); - g_total_param_size += ((child_kernel_arg_size + 255) / 256 * 256); - DEV_RUNTIME_REPORT("child kernel arg size total " << child_kernel_arg_size << ", parameter buffer allocated at " << param_buffer); - if(g_total_param_size > g_max_total_param_size) - g_max_total_param_size = g_total_param_size; - - //store param buffer address and launch config - device_launch_config_t device_launch_config(grid_dim, block_dim, shared_mem, child_kernel_entry); - assert(g_cuda_device_launch_param_map.find(param_buffer) == g_cuda_device_launch_param_map.end()); - g_cuda_device_launch_param_map[param_buffer] = device_launch_config; - - //copy the buffer address to retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 - const symbol *formal_return = target_func->get_return_var(); //void * - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaGetParameterBufferV2 return value has size of " << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && return_size == sizeof(void *)); - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, ¶m_buffer, NULL, NULL); - + } + + // get total child kernel argument size and malloc buffer in global memory + unsigned child_kernel_arg_size = child_kernel_entry->get_args_aligned_size(); + void *param_buffer = thread->get_gpu()->gpu_malloc(child_kernel_arg_size); + g_total_param_size += ((child_kernel_arg_size + 255) / 256 * 256); + DEV_RUNTIME_REPORT("child kernel arg size total " + << child_kernel_arg_size + << ", parameter buffer allocated at " << param_buffer); + if (g_total_param_size > g_max_total_param_size) + g_max_total_param_size = g_total_param_size; + + // store param buffer address and launch config + device_launch_config_t device_launch_config(grid_dim, block_dim, shared_mem, + child_kernel_entry); + assert(g_cuda_device_launch_param_map.find(param_buffer) == + g_cuda_device_launch_param_map.end()); + g_cuda_device_launch_param_map[param_buffer] = device_launch_config; + + // copy the buffer address to retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 + const symbol *formal_return = target_func->get_return_var(); // void * + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaGetParameterBufferV2 return value has size of " + << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && + return_size == sizeof(void *)); + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, ¶m_buffer, NULL, + NULL); } -//Handling device runtime api: -//cudaError_t cudaLaunchDeviceV2(void *parameterBuffer, cudaStream_t stream) -void cuda_device_runtime::gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) { - DEV_RUNTIME_REPORT("Calling cudaLaunchDeviceV2"); - - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert( n_args == 2 ); - - kernel_info_t * device_grid = NULL; - function_info * device_kernel_entry = NULL; - void * parameter_buffer; - struct CUstream_st * child_stream; - device_launch_config_t config; - device_launch_operation_t device_launch_op; - - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# - const symbol *formal_param = target_func->get_arg(arg); //cudaLaunchDeviceV2_param_# - unsigned size=formal_param->get_size_in_bytes(); - assert( formal_param->is_param_local() ); - assert( actual_param_op.is_param_local() ); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - - if(arg == 0) {//paramter buffer for child kernel (in global memory) - //get parameter_buffer from the cudaLaunchDeviceV2_param0 - assert(size == sizeof(void *)); - thread->m_local_mem->read(from_addr, size, ¶meter_buffer); - assert((size_t)parameter_buffer >= GLOBAL_HEAP_START); - DEV_RUNTIME_REPORT("Parameter buffer locating at global memory " << parameter_buffer); - - //get child grid info through parameter_buffer address - assert(g_cuda_device_launch_param_map.find(parameter_buffer) != g_cuda_device_launch_param_map.end()); - config = g_cuda_device_launch_param_map[parameter_buffer]; - //device_grid = op.grid; - device_kernel_entry = config.entry; - DEV_RUNTIME_REPORT("find device kernel " << device_kernel_entry->get_name()); - - //PDOM analysis is done for Parent kernel but not for child kernel. - if (device_kernel_entry->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", device_kernel_entry->get_name().c_str() ); - } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", device_kernel_entry->get_name().c_str() ); - /* - * Some of the instructions like printf() gives the gpgpusim the wrong impression that it is a function call. - * As printf() doesnt have a body like functions do, doing pdom analysis for printf() causes a crash. - */ - if (device_kernel_entry->get_function_size() >0) - device_kernel_entry->do_pdom(); - device_kernel_entry->set_pdom(); - } - - //copy data in parameter_buffer to device kernel param memory - unsigned device_kernel_arg_size = device_kernel_entry->get_args_aligned_size(); - DEV_RUNTIME_REPORT("device_kernel_arg_size " << device_kernel_arg_size); - memory_space *device_kernel_param_mem; - - //create child kernel_info_t and index it with parameter_buffer address - gpgpu_t* gpu=thread->get_gpu(); - device_grid = new kernel_info_t(config.grid_dim, config.block_dim, device_kernel_entry, gpu->getNameArrayMapping(), gpu->getNameInfoMapping()); - device_grid->launch_cycle = gpu->gpu_sim_cycle + gpu->gpu_tot_sim_cycle; - kernel_info_t & parent_grid = thread->get_kernel(); - DEV_RUNTIME_REPORT("child kernel launched by " << parent_grid.name() << ", cta (" << - thread->get_ctaid().x << ", " << thread->get_ctaid().y << ", " << thread->get_ctaid().z << - "), thread (" << thread->get_tid().x << ", " << thread->get_tid().y << ", " << thread->get_tid().z << - ")"); - device_grid->set_parent(&parent_grid, thread->get_ctaid(), thread->get_tid()); - device_launch_op = device_launch_operation_t(device_grid, NULL); - device_kernel_param_mem = device_grid->get_param_memory(); //kernel param - size_t param_start_address = 0; - //copy in word - for(unsigned n = 0; n < device_kernel_arg_size; n += 4) { - unsigned int oneword; - thread->get_gpu()->get_global_memory()->read((size_t)parameter_buffer + n, 4, &oneword); - device_kernel_param_mem->write(param_start_address + n, 4, &oneword, NULL, NULL); - } - } - else if(arg == 1) { //cudaStream for the child kernel - - assert(size == sizeof(cudaStream_t)); - thread->m_local_mem->read(from_addr, size, &child_stream); - - kernel_info_t & parent_kernel = thread->get_kernel(); - if(child_stream == 0) { //default stream on device for current CTA - child_stream = parent_kernel.get_default_stream_cta(thread->get_ctaid()); - DEV_RUNTIME_REPORT("launching child kernel " << device_grid->get_uid() << - " to default stream of the cta " << child_stream->get_uid() << ": " << child_stream); - } - else { - assert(parent_kernel.cta_has_stream(thread->get_ctaid(), child_stream)); - DEV_RUNTIME_REPORT("launching child kernel " << device_grid->get_uid() << - " to stream " << child_stream->get_uid() << ": " << child_stream); - } - - device_launch_op.stream = child_stream; - } - +// Handling device runtime api: +// cudaError_t cudaLaunchDeviceV2(void *parameterBuffer, cudaStream_t stream) +void cuda_device_runtime::gpgpusim_cuda_launchDeviceV2( + const ptx_instruction *pI, ptx_thread_info *thread, + const function_info *target_func) { + DEV_RUNTIME_REPORT("Calling cudaLaunchDeviceV2"); + + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert(n_args == 2); + + kernel_info_t *device_grid = NULL; + function_info *device_kernel_entry = NULL; + void *parameter_buffer; + struct CUstream_st *child_stream; + device_launch_config_t config; + device_launch_operation_t device_launch_op; + + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); // param# + const symbol *formal_param = + target_func->get_arg(arg); // cudaLaunchDeviceV2_param_# + unsigned size = formal_param->get_size_in_bytes(); + assert(formal_param->is_param_local()); + assert(actual_param_op.is_param_local()); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + + if (arg == 0) { // paramter buffer for child kernel (in global memory) + // get parameter_buffer from the cudaLaunchDeviceV2_param0 + assert(size == sizeof(void *)); + thread->m_local_mem->read(from_addr, size, ¶meter_buffer); + assert((size_t)parameter_buffer >= GLOBAL_HEAP_START); + DEV_RUNTIME_REPORT("Parameter buffer locating at global memory " + << parameter_buffer); + + // get child grid info through parameter_buffer address + assert(g_cuda_device_launch_param_map.find(parameter_buffer) != + g_cuda_device_launch_param_map.end()); + config = g_cuda_device_launch_param_map[parameter_buffer]; + // device_grid = op.grid; + device_kernel_entry = config.entry; + DEV_RUNTIME_REPORT("find device kernel " + << device_kernel_entry->get_name()); + + // PDOM analysis is done for Parent kernel but not for child kernel. + if (device_kernel_entry->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", + device_kernel_entry->get_name().c_str()); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", + device_kernel_entry->get_name().c_str()); + /* + * Some of the instructions like printf() gives the gpgpusim the wrong + * impression that it is a function call. + * As printf() doesnt have a body like functions do, doing pdom analysis + * for printf() causes a crash. + */ + if (device_kernel_entry->get_function_size() > 0) + device_kernel_entry->do_pdom(); + device_kernel_entry->set_pdom(); + } + + // copy data in parameter_buffer to device kernel param memory + unsigned device_kernel_arg_size = + device_kernel_entry->get_args_aligned_size(); + DEV_RUNTIME_REPORT("device_kernel_arg_size " << device_kernel_arg_size); + memory_space *device_kernel_param_mem; + + // create child kernel_info_t and index it with parameter_buffer address + gpgpu_t *gpu = thread->get_gpu(); + device_grid = new kernel_info_t( + config.grid_dim, config.block_dim, device_kernel_entry, + gpu->getNameArrayMapping(), gpu->getNameInfoMapping()); + device_grid->launch_cycle = gpu->gpu_sim_cycle + gpu->gpu_tot_sim_cycle; + kernel_info_t &parent_grid = thread->get_kernel(); + DEV_RUNTIME_REPORT( + "child kernel launched by " + << parent_grid.name() << ", cta (" << thread->get_ctaid().x << ", " + << thread->get_ctaid().y << ", " << thread->get_ctaid().z + << "), thread (" << thread->get_tid().x << ", " << thread->get_tid().y + << ", " << thread->get_tid().z << ")"); + device_grid->set_parent(&parent_grid, thread->get_ctaid(), + thread->get_tid()); + device_launch_op = device_launch_operation_t(device_grid, NULL); + device_kernel_param_mem = device_grid->get_param_memory(); // kernel + // param + size_t param_start_address = 0; + // copy in word + for (unsigned n = 0; n < device_kernel_arg_size; n += 4) { + unsigned int oneword; + thread->get_gpu()->get_global_memory()->read( + (size_t)parameter_buffer + n, 4, &oneword); + device_kernel_param_mem->write(param_start_address + n, 4, &oneword, + NULL, NULL); + } + } else if (arg == 1) { // cudaStream for the child kernel + + assert(size == sizeof(cudaStream_t)); + thread->m_local_mem->read(from_addr, size, &child_stream); + + kernel_info_t &parent_kernel = thread->get_kernel(); + if (child_stream == 0) { // default stream on device for current CTA + child_stream = + parent_kernel.get_default_stream_cta(thread->get_ctaid()); + DEV_RUNTIME_REPORT("launching child kernel " + << device_grid->get_uid() + << " to default stream of the cta " + << child_stream->get_uid() << ": " << child_stream); + } else { + assert(parent_kernel.cta_has_stream(thread->get_ctaid(), child_stream)); + DEV_RUNTIME_REPORT("launching child kernel " + << device_grid->get_uid() << " to stream " + << child_stream->get_uid() << ": " << child_stream); + } + + device_launch_op.stream = child_stream; } - - - //launch child kernel - g_cuda_device_launch_op.push_back(device_launch_op); - g_cuda_device_launch_param_map.erase(parameter_buffer); - - //set retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 - const symbol *formal_return = target_func->get_return_var(); //cudaError_t - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaLaunchDeviceV2 return value has size of " << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size - && return_size == sizeof(cudaError_t)); - cudaError_t error = cudaSuccess; - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); - + } + + // launch child kernel + g_cuda_device_launch_op.push_back(device_launch_op); + g_cuda_device_launch_param_map.erase(parameter_buffer); + + // set retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 + const symbol *formal_return = target_func->get_return_var(); // cudaError_t + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaLaunchDeviceV2 return value has size of " + << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && + return_size == sizeof(cudaError_t)); + cudaError_t error = cudaSuccess; + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); } - -//Handling device runtime api: -//cudaError_t cudaStreamCreateWithFlags ( cudaStream_t* pStream, unsigned int flags) -//flags can only be cudaStreamNonBlocking -void cuda_device_runtime::gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) { - DEV_RUNTIME_REPORT("Calling cudaStreamCreateWithFlags"); - - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert( n_args == 2 ); - - size_t generic_pStream_addr; - addr_t pStream_addr; - unsigned int flags; - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# - const symbol *formal_param = target_func->get_arg(arg); //cudaStreamCreateWithFlags_param_# - unsigned size=formal_param->get_size_in_bytes(); - assert( formal_param->is_param_local() ); - assert( actual_param_op.is_param_local() ); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - - if(arg == 0) {//cudaStream_t * pStream, address of cudaStream_t - assert(size == sizeof(cudaStream_t *)); - thread->m_local_mem->read(from_addr, size, &generic_pStream_addr); - - //pStream should be non-zero address in local memory - pStream_addr = generic_to_local(thread->get_hw_sid(), thread->get_hw_tid(), generic_pStream_addr); - - DEV_RUNTIME_REPORT("pStream locating at local memory " << pStream_addr); - } - else if(arg == 1) { //unsigned int flags, should be cudaStreamNonBlocking - assert(size == sizeof(unsigned int)); - thread->m_local_mem->read(from_addr, size, &flags); - assert(flags == cudaStreamNonBlocking); - } +// Handling device runtime api: +// cudaError_t cudaStreamCreateWithFlags ( cudaStream_t* pStream, unsigned int +// flags) +// flags can only be cudaStreamNonBlocking +void cuda_device_runtime::gpgpusim_cuda_streamCreateWithFlags( + const ptx_instruction *pI, ptx_thread_info *thread, + const function_info *target_func) { + DEV_RUNTIME_REPORT("Calling cudaStreamCreateWithFlags"); + + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert(n_args == 2); + + size_t generic_pStream_addr; + addr_t pStream_addr; + unsigned int flags; + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); // param# + const symbol *formal_param = + target_func->get_arg(arg); // cudaStreamCreateWithFlags_param_# + unsigned size = formal_param->get_size_in_bytes(); + assert(formal_param->is_param_local()); + assert(actual_param_op.is_param_local()); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + + if (arg == 0) { // cudaStream_t * pStream, address of cudaStream_t + assert(size == sizeof(cudaStream_t *)); + thread->m_local_mem->read(from_addr, size, &generic_pStream_addr); + + // pStream should be non-zero address in local memory + pStream_addr = generic_to_local( + thread->get_hw_sid(), thread->get_hw_tid(), generic_pStream_addr); + + DEV_RUNTIME_REPORT("pStream locating at local memory " << pStream_addr); + } else if (arg == + 1) { // unsigned int flags, should be cudaStreamNonBlocking + assert(size == sizeof(unsigned int)); + thread->m_local_mem->read(from_addr, size, &flags); + assert(flags == cudaStreamNonBlocking); } - - //create stream and write back to param0 - CUstream_st * stream = thread->get_kernel().create_stream_cta(thread->get_ctaid()); - DEV_RUNTIME_REPORT("Create stream " << stream->get_uid() << ": " << stream); - thread->m_local_mem->write(pStream_addr, sizeof(cudaStream_t), &stream, NULL, NULL); - - //set retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 - const symbol *formal_return = target_func->get_return_var(); //cudaError_t - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaStreamCreateWithFlags return value has size of " << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size - && return_size == sizeof(cudaError_t)); - cudaError_t error = cudaSuccess; - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); - + } + + // create stream and write back to param0 + CUstream_st *stream = + thread->get_kernel().create_stream_cta(thread->get_ctaid()); + DEV_RUNTIME_REPORT("Create stream " << stream->get_uid() << ": " << stream); + thread->m_local_mem->write(pStream_addr, sizeof(cudaStream_t), &stream, NULL, + NULL); + + // set retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 + const symbol *formal_return = target_func->get_return_var(); // cudaError_t + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaStreamCreateWithFlags return value has size of " + << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && + return_size == sizeof(cudaError_t)); + cudaError_t error = cudaSuccess; + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); } - void cuda_device_runtime::launch_one_device_kernel() { - if(!g_cuda_device_launch_op.empty()) { - device_launch_operation_t &op = g_cuda_device_launch_op.front(); - - stream_operation stream_op = stream_operation(op.grid, gpgpu_ctx->func_sim->g_ptx_sim_mode, op.stream); - gpgpu_ctx->the_gpgpusim->g_stream_manager->push(stream_op); - g_cuda_device_launch_op.pop_front(); - } + if (!g_cuda_device_launch_op.empty()) { + device_launch_operation_t &op = g_cuda_device_launch_op.front(); + + stream_operation stream_op = stream_operation( + op.grid, gpgpu_ctx->func_sim->g_ptx_sim_mode, op.stream); + gpgpu_ctx->the_gpgpusim->g_stream_manager->push(stream_op); + g_cuda_device_launch_op.pop_front(); + } } void cuda_device_runtime::launch_all_device_kernels() { - while(!g_cuda_device_launch_op.empty()) { - launch_one_device_kernel(); - } + while (!g_cuda_device_launch_op.empty()) { + launch_one_device_kernel(); + } } #endif diff --git a/src/cuda-sim/cuda_device_runtime.h b/src/cuda-sim/cuda_device_runtime.h index 7f7a0ca..1d661b2 100644 --- a/src/cuda-sim/cuda_device_runtime.h +++ b/src/cuda-sim/cuda_device_runtime.h @@ -1,67 +1,66 @@ #ifndef __cuda_device_runtime_h__ #define __cuda_device_runtime_h__ -//Jin: cuda_device_runtime.h -//Defines CUDA device runtime APIs for CDP support +// Jin: cuda_device_runtime.h +// Defines CUDA device runtime APIs for CDP support class device_launch_config_t { + public: + device_launch_config_t() {} -public: - device_launch_config_t() {} - - device_launch_config_t(dim3 _grid_dim, - dim3 _block_dim, - unsigned int _shared_mem, - function_info * _entry): - grid_dim(_grid_dim), - block_dim(_block_dim), - shared_mem(_shared_mem), - entry(_entry) {} - - dim3 grid_dim; - dim3 block_dim; - unsigned int shared_mem; - function_info * entry; + device_launch_config_t(dim3 _grid_dim, dim3 _block_dim, + unsigned int _shared_mem, function_info* _entry) + : grid_dim(_grid_dim), + block_dim(_block_dim), + shared_mem(_shared_mem), + entry(_entry) {} + dim3 grid_dim; + dim3 block_dim; + unsigned int shared_mem; + function_info* entry; }; class device_launch_operation_t { + public: + device_launch_operation_t() {} + device_launch_operation_t(kernel_info_t* _grid, CUstream_st* _stream) + : grid(_grid), stream(_stream) {} -public: - device_launch_operation_t() {} - device_launch_operation_t(kernel_info_t *_grid, - CUstream_st * _stream) : - grid(_grid), stream(_stream) {} - - kernel_info_t * grid; //a new child grid - - CUstream_st * stream; + kernel_info_t* grid; // a new child grid + CUstream_st* stream; }; class gpgpu_context; class cuda_device_runtime { - public: - cuda_device_runtime( gpgpu_context* ctx ) { - g_total_param_size = 0; - g_max_total_param_size = 0; - gpgpu_ctx = ctx; - } - unsigned long long g_total_param_size; - std::map g_cuda_device_launch_param_map; - std::list g_cuda_device_launch_op; - unsigned g_kernel_launch_latency; - unsigned long long g_max_total_param_size; - bool g_cdp_enabled; + public: + cuda_device_runtime(gpgpu_context* ctx) { + g_total_param_size = 0; + g_max_total_param_size = 0; + gpgpu_ctx = ctx; + } + unsigned long long g_total_param_size; + std::map g_cuda_device_launch_param_map; + std::list g_cuda_device_launch_op; + unsigned g_kernel_launch_latency; + unsigned long long g_max_total_param_size; + bool g_cdp_enabled; - // backward pointer - class gpgpu_context* gpgpu_ctx; + // backward pointer + class gpgpu_context* gpgpu_ctx; #if (CUDART_VERSION >= 5000) #pragma once - void gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); - void gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); - void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); - void launch_all_device_kernels(); - void launch_one_device_kernel(); + void gpgpusim_cuda_launchDeviceV2(const ptx_instruction* pI, + ptx_thread_info* thread, + const function_info* target_func); + void gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction* pI, + ptx_thread_info* thread, + const function_info* target_func); + void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction* pI, + ptx_thread_info* thread, + const function_info* target_func); + void launch_all_device_kernels(); + void launch_one_device_kernel(); #endif }; diff --git a/src/cuda-sim/half.h b/src/cuda-sim/half.h index 9f74bb7..d33b03c 100644 --- a/src/cuda-sim/half.h +++ b/src/cuda-sim/half.h @@ -2,17 +2,25 @@ // // Copyright (c) 2012-2017 Christian Rau // -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation -// files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, -// modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation +// files (the "Software"), to deal in the Software without restriction, +// including without limitation the rights to use, copy, +// modify, merge, publish, distribute, sublicense, and/or sell copies of the +// Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // -// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. // -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE +// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. // Version 1.12.0 @@ -23,180 +31,191 @@ #define HALF_HALF_HPP /// Combined gcc version number. -#define HALF_GNUC_VERSION (__GNUC__*100+__GNUC_MINOR__) - -//check C++11 language features -#if defined(__clang__) //clang - #if __has_feature(cxx_static_assert) && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) - #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 - #endif - #if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR) - #define HALF_ENABLE_CPP11_CONSTEXPR 1 - #endif - #if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT) - #define HALF_ENABLE_CPP11_NOEXCEPT 1 - #endif - #if __has_feature(cxx_user_literals) && !defined(HALF_ENABLE_CPP11_USER_LITERALS) - #define HALF_ENABLE_CPP11_USER_LITERALS 1 - #endif - #if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && !defined(HALF_ENABLE_CPP11_LONG_LONG) - #define HALF_ENABLE_CPP11_LONG_LONG 1 - #endif -/*#elif defined(__INTEL_COMPILER) //Intel C++ - #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) ???????? - #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 - #endif - #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) ???????? - #define HALF_ENABLE_CPP11_CONSTEXPR 1 - #endif - #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) ???????? - #define HALF_ENABLE_CPP11_NOEXCEPT 1 - #endif - #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_LONG_LONG) ???????? - #define HALF_ENABLE_CPP11_LONG_LONG 1 - #endif*/ -#elif defined(__GNUC__) //gcc - #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L - #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) - #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 - #endif - #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) - #define HALF_ENABLE_CPP11_CONSTEXPR 1 - #endif - #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) - #define HALF_ENABLE_CPP11_NOEXCEPT 1 - #endif - #if HALF_GNUC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) - #define HALF_ENABLE_CPP11_USER_LITERALS 1 - #endif - #if !defined(HALF_ENABLE_CPP11_LONG_LONG) - #define HALF_ENABLE_CPP11_LONG_LONG 1 - #endif - #endif -#elif defined(_MSC_VER) //Visual C++ - #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) - #define HALF_ENABLE_CPP11_CONSTEXPR 1 - #endif - #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) - #define HALF_ENABLE_CPP11_NOEXCEPT 1 - #endif - #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) - #define HALF_ENABLE_CPP11_USER_LITERALS 1 - #endif - #if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) - #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 - #endif - #if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG) - #define HALF_ENABLE_CPP11_LONG_LONG 1 - #endif - #define HALF_POP_WARNINGS 1 - #pragma warning(push) - #pragma warning(disable : 4099 4127 4146) //struct vs class, constant in if, negative unsigned -#endif - -//check C++11 library features +#define HALF_GNUC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +// check C++11 language features +#if defined(__clang__) // clang +#if __has_feature(cxx_static_assert) && \ + !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) +#define HALF_ENABLE_CPP11_STATIC_ASSERT 1 +#endif +#if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR) +#define HALF_ENABLE_CPP11_CONSTEXPR 1 +#endif +#if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT) +#define HALF_ENABLE_CPP11_NOEXCEPT 1 +#endif +#if __has_feature(cxx_user_literals) && \ + !defined(HALF_ENABLE_CPP11_USER_LITERALS) +#define HALF_ENABLE_CPP11_USER_LITERALS 1 +#endif +#if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && \ + !defined(HALF_ENABLE_CPP11_LONG_LONG) +#define HALF_ENABLE_CPP11_LONG_LONG 1 +#endif +/*#elif defined(__INTEL_COMPILER) + //Intel C++ + #if __INTEL_COMPILER >= 1100 && + !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) ???????? + #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 + #endif + #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) + ???????? + #define HALF_ENABLE_CPP11_CONSTEXPR 1 + #endif + #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) + ???????? + #define HALF_ENABLE_CPP11_NOEXCEPT 1 + #endif + #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_LONG_LONG) + ???????? + #define HALF_ENABLE_CPP11_LONG_LONG 1 + #endif*/ +#elif defined(__GNUC__) // gcc +#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L +#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) +#define HALF_ENABLE_CPP11_STATIC_ASSERT 1 +#endif +#if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) +#define HALF_ENABLE_CPP11_CONSTEXPR 1 +#endif +#if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) +#define HALF_ENABLE_CPP11_NOEXCEPT 1 +#endif +#if HALF_GNUC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) +#define HALF_ENABLE_CPP11_USER_LITERALS 1 +#endif +#if !defined(HALF_ENABLE_CPP11_LONG_LONG) +#define HALF_ENABLE_CPP11_LONG_LONG 1 +#endif +#endif +#elif defined(_MSC_VER) // Visual C++ +#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) +#define HALF_ENABLE_CPP11_CONSTEXPR 1 +#endif +#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) +#define HALF_ENABLE_CPP11_NOEXCEPT 1 +#endif +#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) +#define HALF_ENABLE_CPP11_USER_LITERALS 1 +#endif +#if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) +#define HALF_ENABLE_CPP11_STATIC_ASSERT 1 +#endif +#if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG) +#define HALF_ENABLE_CPP11_LONG_LONG 1 +#endif +#define HALF_POP_WARNINGS 1 +#pragma warning(push) +#pragma warning(disable : 4099 4127 4146) // struct vs class, constant in if, + // negative unsigned +#endif + +// check C++11 library features #include -#if defined(_LIBCPP_VERSION) //libc++ - #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 - #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS - #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 - #endif - #ifndef HALF_ENABLE_CPP11_CSTDINT - #define HALF_ENABLE_CPP11_CSTDINT 1 - #endif - #ifndef HALF_ENABLE_CPP11_CMATH - #define HALF_ENABLE_CPP11_CMATH 1 - #endif - #ifndef HALF_ENABLE_CPP11_HASH - #define HALF_ENABLE_CPP11_HASH 1 - #endif - #endif -#elif defined(__GLIBCXX__) //libstdc++ - #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 - #ifdef __clang__ - #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) - #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 - #endif - #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT) - #define HALF_ENABLE_CPP11_CSTDINT 1 - #endif - #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH) - #define HALF_ENABLE_CPP11_CMATH 1 - #endif - #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH) - #define HALF_ENABLE_CPP11_HASH 1 - #endif - #else - #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT) - #define HALF_ENABLE_CPP11_CSTDINT 1 - #endif - #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH) - #define HALF_ENABLE_CPP11_CMATH 1 - #endif - #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH) - #define HALF_ENABLE_CPP11_HASH 1 - #endif - #endif - #endif -#elif defined(_CPPLIB_VER) //Dinkumware/Visual C++ - #if _CPPLIB_VER >= 520 - #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS - #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 - #endif - #ifndef HALF_ENABLE_CPP11_CSTDINT - #define HALF_ENABLE_CPP11_CSTDINT 1 - #endif - #ifndef HALF_ENABLE_CPP11_HASH - #define HALF_ENABLE_CPP11_HASH 1 - #endif - #endif - #if _CPPLIB_VER >= 610 - #ifndef HALF_ENABLE_CPP11_CMATH - #define HALF_ENABLE_CPP11_CMATH 1 - #endif - #endif +#if defined(_LIBCPP_VERSION) // libc++ +#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 +#ifndef HALF_ENABLE_CPP11_TYPE_TRAITS +#define HALF_ENABLE_CPP11_TYPE_TRAITS 1 +#endif +#ifndef HALF_ENABLE_CPP11_CSTDINT +#define HALF_ENABLE_CPP11_CSTDINT 1 +#endif +#ifndef HALF_ENABLE_CPP11_CMATH +#define HALF_ENABLE_CPP11_CMATH 1 +#endif +#ifndef HALF_ENABLE_CPP11_HASH +#define HALF_ENABLE_CPP11_HASH 1 +#endif +#endif +#elif defined(__GLIBCXX__) // libstdc++ +#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 +#ifdef __clang__ +#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) +#define HALF_ENABLE_CPP11_TYPE_TRAITS 1 +#endif +#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT) +#define HALF_ENABLE_CPP11_CSTDINT 1 +#endif +#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH) +#define HALF_ENABLE_CPP11_CMATH 1 +#endif +#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH) +#define HALF_ENABLE_CPP11_HASH 1 +#endif +#else +#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT) +#define HALF_ENABLE_CPP11_CSTDINT 1 +#endif +#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH) +#define HALF_ENABLE_CPP11_CMATH 1 +#endif +#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH) +#define HALF_ENABLE_CPP11_HASH 1 +#endif +#endif +#endif +#elif defined(_CPPLIB_VER) // Dinkumware/Visual C++ +#if _CPPLIB_VER >= 520 +#ifndef HALF_ENABLE_CPP11_TYPE_TRAITS +#define HALF_ENABLE_CPP11_TYPE_TRAITS 1 +#endif +#ifndef HALF_ENABLE_CPP11_CSTDINT +#define HALF_ENABLE_CPP11_CSTDINT 1 +#endif +#ifndef HALF_ENABLE_CPP11_HASH +#define HALF_ENABLE_CPP11_HASH 1 +#endif +#endif +#if _CPPLIB_VER >= 610 +#ifndef HALF_ENABLE_CPP11_CMATH +#define HALF_ENABLE_CPP11_CMATH 1 +#endif +#endif #endif #undef HALF_GNUC_VERSION -//support constexpr +// support constexpr #if HALF_ENABLE_CPP11_CONSTEXPR - #define HALF_CONSTEXPR constexpr - #define HALF_CONSTEXPR_CONST constexpr +#define HALF_CONSTEXPR constexpr +#define HALF_CONSTEXPR_CONST constexpr #else - #define HALF_CONSTEXPR - #define HALF_CONSTEXPR_CONST const +#define HALF_CONSTEXPR +#define HALF_CONSTEXPR_CONST const #endif -//support noexcept +// support noexcept #if HALF_ENABLE_CPP11_NOEXCEPT - #define HALF_NOEXCEPT noexcept - #define HALF_NOTHROW noexcept +#define HALF_NOEXCEPT noexcept +#define HALF_NOTHROW noexcept #else - #define HALF_NOEXCEPT - #define HALF_NOTHROW throw() +#define HALF_NOEXCEPT +#define HALF_NOTHROW throw() #endif #include -#include -#include #include #include #include +#include +#include #if HALF_ENABLE_CPP11_TYPE_TRAITS - #include +#include #endif #if HALF_ENABLE_CPP11_CSTDINT - #include +#include #endif #if HALF_ENABLE_CPP11_HASH - #include +#include #endif - /// Default rounding mode. -/// This specifies the rounding mode used for all conversions between [half](\ref half_float::half)s and `float`s as well as -/// for the half_cast() if not specifying a rounding mode explicitly. It can be redefined (before including half.hpp) to one -/// of the standard rounding modes using their respective constants or the equivalent values of `std::float_round_style`: +/// This specifies the rounding mode used for all conversions between +/// [half](\ref half_float::half)s and `float`s as well as +/// for the half_cast() if not specifying a rounding mode explicitly. It can be +/// redefined (before including half.hpp) to one +/// of the standard rounding modes using their respective constants or the +/// equivalent values of `std::float_round_style`: /// /// `std::float_round_style` | value | rounding /// ---------------------------------|-------|------------------------- @@ -206,256 +225,354 @@ /// `std::round_toward_infinity` | 2 | toward positive infinity /// `std::round_toward_neg_infinity` | 3 | toward negative infinity /// -/// By default this is set to `-1` (`std::round_indeterminate`), which uses truncation (round toward zero, but with overflows -/// set to infinity) and is the fastest rounding mode possible. It can even be set to `std::numeric_limits::round_style` -/// to synchronize the rounding mode with that of the underlying single-precision implementation. +/// By default this is set to `-1` (`std::round_indeterminate`), which uses +/// truncation (round toward zero, but with overflows +/// set to infinity) and is the fastest rounding mode possible. It can even be +/// set to `std::numeric_limits::round_style` +/// to synchronize the rounding mode with that of the underlying +/// single-precision implementation. #ifndef HALF_ROUND_STYLE - #define HALF_ROUND_STYLE -1 // = std::round_indeterminate +#define HALF_ROUND_STYLE -1 // = std::round_indeterminate #endif /// Tie-breaking behaviour for round to nearest. -/// This specifies if ties in round to nearest should be resolved by rounding to the nearest even value. By default this is -/// defined to `0` resulting in the faster but slightly more biased behaviour of rounding away from zero in half-way cases (and -/// thus equal to the round() function), but can be redefined to `1` (before including half.hpp) if more IEEE-conformant +/// This specifies if ties in round to nearest should be resolved by rounding to +/// the nearest even value. By default this is +/// defined to `0` resulting in the faster but slightly more biased behaviour of +/// rounding away from zero in half-way cases (and +/// thus equal to the round() function), but can be redefined to `1` (before +/// including half.hpp) if more IEEE-conformant /// behaviour is needed. #ifndef HALF_ROUND_TIES_TO_EVEN - #define HALF_ROUND_TIES_TO_EVEN 0 // ties away from zero +#define HALF_ROUND_TIES_TO_EVEN 0 // ties away from zero #endif /// Value signaling overflow. -/// In correspondence with `HUGE_VAL[F|L]` from `` this symbol expands to a positive value signaling the overflow of an +/// In correspondence with `HUGE_VAL[F|L]` from `` this symbol expands to +/// a positive value signaling the overflow of an /// operation, in particular it just evaluates to positive infinity. -#define HUGE_VALH std::numeric_limits::infinity() +#define HUGE_VALH std::numeric_limits::infinity() /// Fast half-precision fma function. -/// This symbol is only defined if the fma() function generally executes as fast as, or faster than, a separate -/// half-precision multiplication followed by an addition. Due to the internal single-precision implementation of all +/// This symbol is only defined if the fma() function generally executes as fast +/// as, or faster than, a separate +/// half-precision multiplication followed by an addition. Due to the internal +/// single-precision implementation of all /// arithmetic operations, this is in fact always the case. -#define FP_FAST_FMAH 1 +#define FP_FAST_FMAH 1 #ifndef FP_ILOGB0 - #define FP_ILOGB0 INT_MIN +#define FP_ILOGB0 INT_MIN #endif #ifndef FP_ILOGBNAN - #define FP_ILOGBNAN INT_MAX +#define FP_ILOGBNAN INT_MAX #endif #ifndef FP_SUBNORMAL - #define FP_SUBNORMAL 0 +#define FP_SUBNORMAL 0 #endif #ifndef FP_ZERO - #define FP_ZERO 1 +#define FP_ZERO 1 #endif #ifndef FP_NAN - #define FP_NAN 2 +#define FP_NAN 2 #endif #ifndef FP_INFINITE - #define FP_INFINITE 3 +#define FP_INFINITE 3 #endif #ifndef FP_NORMAL - #define FP_NORMAL 4 +#define FP_NORMAL 4 #endif - /// Main namespace for half precision functionality. /// This namespace contains all the functionality provided by the library. -namespace half_float -{ - class half; +namespace half_float { +class half; #if HALF_ENABLE_CPP11_USER_LITERALS - /// Library-defined half-precision literals. - /// Import this namespace to enable half-precision floating point literals: - /// ~~~~{.cpp} - /// using namespace half_float::literal; - /// half_float::half = 4.2_h; - /// ~~~~ - namespace literal - { - half operator"" _h(long double); - } -#endif - - /// \internal - /// \brief Implementation details. - namespace detail - { - #if HALF_ENABLE_CPP11_TYPE_TRAITS - /// Conditional type. - template struct conditional : std::conditional {}; - - /// Helper for tag dispatching. - template struct bool_type : std::integral_constant {}; - using std::true_type; - using std::false_type; - - /// Type traits for floating point types. - template struct is_float : std::is_floating_point {}; - #else - /// Conditional type. - template struct conditional { typedef T type; }; - template struct conditional { typedef F type; }; - - /// Helper for tag dispatching. - template struct bool_type {}; - typedef bool_type true_type; - typedef bool_type false_type; - - /// Type traits for floating point types. - template struct is_float : false_type {}; - template struct is_float : is_float {}; - template struct is_float : is_float {}; - template struct is_float : is_float {}; - template<> struct is_float : true_type {}; - template<> struct is_float : true_type {}; - template<> struct is_float : true_type {}; - #endif - - /// Type traits for floating point bits. - template struct bits { typedef unsigned char type; }; - template struct bits : bits {}; - template struct bits : bits {}; - template struct bits : bits {}; - - #if HALF_ENABLE_CPP11_CSTDINT - /// Unsigned integer of (at least) 16 bits width. - typedef std::uint_least16_t uint16; - - /// Unsigned integer of (at least) 32 bits width. - template<> struct bits { typedef std::uint_least32_t type; }; - - /// Unsigned integer of (at least) 64 bits width. - template<> struct bits { typedef std::uint_least64_t type; }; - #else - /// Unsigned integer of (at least) 16 bits width. - typedef unsigned short uint16; - - /// Unsigned integer of (at least) 32 bits width. - template<> struct bits : conditional::digits>=32,unsigned int,unsigned long> {}; - - #if HALF_ENABLE_CPP11_LONG_LONG - /// Unsigned integer of (at least) 64 bits width. - template<> struct bits : conditional::digits>=64,unsigned long,unsigned long long> {}; - #else - /// Unsigned integer of (at least) 64 bits width. - template<> struct bits { typedef unsigned long type; }; - #endif - #endif - - /// Tag type for binary construction. - struct binary_t {}; - - /// Tag for binary construction. - HALF_CONSTEXPR_CONST binary_t binary = binary_t(); - - /// Temporary half-precision expression. - /// This class represents a half-precision expression which just stores a single-precision value internally. - struct expr - { - /// Conversion constructor. - /// \param f single-precision value to convert - explicit HALF_CONSTEXPR expr(float f) HALF_NOEXCEPT : value_(f) {} - - /// Conversion to single-precision. - /// \return single precision value representing expression value - HALF_CONSTEXPR operator float() const HALF_NOEXCEPT { return value_; } - - private: - /// Internal expression value stored in single-precision. - float value_; - }; - - /// SFINAE helper for generic half-precision functions. - /// This class template has to be specialized for each valid combination of argument types to provide a corresponding - /// `type` member equivalent to \a T. - /// \tparam T type to return - template struct enable {}; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - - /// Return type for specialized generic 2-argument half-precision functions. - /// This class template has to be specialized for each valid combination of argument types to provide a corresponding - /// `type` member denoting the appropriate return type. - /// \tparam T first argument type - /// \tparam U first argument type - template struct result : enable {}; - template<> struct result { typedef half type; }; - - /// \name Classification helpers - /// \{ - - /// Check for infinity. - /// \tparam T argument type (builtin floating point type) - /// \param arg value to query - /// \retval true if infinity - /// \retval false else - template bool builtin_isinf(T arg) - { - #if HALF_ENABLE_CPP11_CMATH - return std::isinf(arg); - #elif defined(_MSC_VER) - return !::_finite(static_cast(arg)) && !::_isnan(static_cast(arg)); - #else - return arg == std::numeric_limits::infinity() || arg == -std::numeric_limits::infinity(); - #endif - } - - /// Check for NaN. - /// \tparam T argument type (builtin floating point type) - /// \param arg value to query - /// \retval true if not a number - /// \retval false else - template bool builtin_isnan(T arg) - { - #if HALF_ENABLE_CPP11_CMATH - return std::isnan(arg); - #elif defined(_MSC_VER) - return ::_isnan(static_cast(arg)) != 0; - #else - return arg != arg; - #endif - } - - /// Check sign. - /// \tparam T argument type (builtin floating point type) - /// \param arg value to query - /// \retval true if signbit set - /// \retval false else - template bool builtin_signbit(T arg) - { - #if HALF_ENABLE_CPP11_CMATH - return std::signbit(arg); - #else - return arg < T() || (arg == T() && T(1)/arg < T()); - #endif - } - - /// \} - /// \name Conversion - /// \{ - - /// Convert IEEE single-precision to half-precision. - /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \param value single-precision value - /// \return binary representation of half-precision value - template uint16 float2half_impl(float value, true_type) - { - typedef bits::type uint32; - uint32 bits;// = *reinterpret_cast(&value); //violating strict aliasing! - std::memcpy(&bits, &value, sizeof(float)); -/* uint16 hbits = (bits>>16) & 0x8000; +/// Library-defined half-precision literals. +/// Import this namespace to enable half-precision floating point literals: +/// ~~~~{.cpp} +/// using namespace half_float::literal; +/// half_float::half = 4.2_h; +/// ~~~~ +namespace literal { +half operator"" _h(long double); +} +#endif + +/// \internal +/// \brief Implementation details. +namespace detail { +#if HALF_ENABLE_CPP11_TYPE_TRAITS +/// Conditional type. +template +struct conditional : std::conditional {}; + +/// Helper for tag dispatching. +template +struct bool_type : std::integral_constant {}; +using std::true_type; +using std::false_type; + +/// Type traits for floating point types. +template +struct is_float : std::is_floating_point {}; +#else +/// Conditional type. +template +struct conditional { + typedef T type; +}; +template +struct conditional { + typedef F type; +}; + +/// Helper for tag dispatching. +template +struct bool_type {}; +typedef bool_type true_type; +typedef bool_type false_type; + +/// Type traits for floating point types. +template +struct is_float : false_type {}; +template +struct is_float : is_float {}; +template +struct is_float : is_float {}; +template +struct is_float : is_float {}; +template <> +struct is_float : true_type {}; +template <> +struct is_float : true_type {}; +template <> +struct is_float : true_type {}; +#endif + +/// Type traits for floating point bits. +template +struct bits { + typedef unsigned char type; +}; +template +struct bits : bits {}; +template +struct bits : bits {}; +template +struct bits : bits {}; + +#if HALF_ENABLE_CPP11_CSTDINT +/// Unsigned integer of (at least) 16 bits width. +typedef std::uint_least16_t uint16; + +/// Unsigned integer of (at least) 32 bits width. +template <> +struct bits { + typedef std::uint_least32_t type; +}; + +/// Unsigned integer of (at least) 64 bits width. +template <> +struct bits { + typedef std::uint_least64_t type; +}; +#else +/// Unsigned integer of (at least) 16 bits width. +typedef unsigned short uint16; + +/// Unsigned integer of (at least) 32 bits width. +template <> +struct bits + : conditional::digits >= 32, unsigned int, + unsigned long> {}; + +#if HALF_ENABLE_CPP11_LONG_LONG +/// Unsigned integer of (at least) 64 bits width. +template <> +struct bits + : conditional::digits >= 64, + unsigned long, unsigned long long> {}; +#else +/// Unsigned integer of (at least) 64 bits width. +template <> +struct bits { + typedef unsigned long type; +}; +#endif +#endif + +/// Tag type for binary construction. +struct binary_t {}; + +/// Tag for binary construction. +HALF_CONSTEXPR_CONST binary_t binary = binary_t(); + +/// Temporary half-precision expression. +/// This class represents a half-precision expression which just stores a +/// single-precision value internally. +struct expr { + /// Conversion constructor. + /// \param f single-precision value to convert + explicit HALF_CONSTEXPR expr(float f) HALF_NOEXCEPT : value_(f) {} + + /// Conversion to single-precision. + /// \return single precision value representing expression value + HALF_CONSTEXPR operator float() const HALF_NOEXCEPT { return value_; } + + private: + /// Internal expression value stored in single-precision. + float value_; +}; + +/// SFINAE helper for generic half-precision functions. +/// This class template has to be specialized for each valid combination of +/// argument types to provide a corresponding +/// `type` member equivalent to \a T. +/// \tparam T type to return +template +struct enable {}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; + +/// Return type for specialized generic 2-argument half-precision functions. +/// This class template has to be specialized for each valid combination of +/// argument types to provide a corresponding +/// `type` member denoting the appropriate return type. +/// \tparam T first argument type +/// \tparam U first argument type +template +struct result : enable {}; +template <> +struct result { + typedef half type; +}; + +/// \name Classification helpers +/// \{ + +/// Check for infinity. +/// \tparam T argument type (builtin floating point type) +/// \param arg value to query +/// \retval true if infinity +/// \retval false else +template +bool builtin_isinf(T arg) { +#if HALF_ENABLE_CPP11_CMATH + return std::isinf(arg); +#elif defined(_MSC_VER) + return !::_finite(static_cast(arg)) && + !::_isnan(static_cast(arg)); +#else + return arg == std::numeric_limits::infinity() || + arg == -std::numeric_limits::infinity(); +#endif +} + +/// Check for NaN. +/// \tparam T argument type (builtin floating point type) +/// \param arg value to query +/// \retval true if not a number +/// \retval false else +template +bool builtin_isnan(T arg) { +#if HALF_ENABLE_CPP11_CMATH + return std::isnan(arg); +#elif defined(_MSC_VER) + return ::_isnan(static_cast(arg)) != 0; +#else + return arg != arg; +#endif +} + +/// Check sign. +/// \tparam T argument type (builtin floating point type) +/// \param arg value to query +/// \retval true if signbit set +/// \retval false else +template +bool builtin_signbit(T arg) { +#if HALF_ENABLE_CPP11_CMATH + return std::signbit(arg); +#else + return arg < T() || (arg == T() && T(1) / arg < T()); +#endif +} + +/// \} +/// \name Conversion +/// \{ + +/// Convert IEEE single-precision to half-precision. +/// Credit for this goes to [Jeroen van der +/// Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding +/// \param value single-precision value +/// \return binary representation of half-precision value +template +uint16 float2half_impl(float value, true_type) { + typedef bits::type uint32; + uint32 + bits; // = *reinterpret_cast(&value); //violating + // strict aliasing! + std::memcpy(&bits, &value, sizeof(float)); + /* uint16 hbits = (bits>>16) & 0x8000; bits &= 0x7FFFFFFF; int exp = bits >> 23; if(exp == 255) @@ -498,2570 +615,3238 @@ namespace half_float hbits += ~(hbits>>15) & (s|g); else if(R == std::round_toward_neg_infinity) hbits += (hbits>>15) & (g|s); -*/ static const uint16 base_table[512] = { - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100, - 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00, 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00, - 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100, - 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00, - 0xC000, 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00, 0xF000, 0xF400, 0xF800, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00 }; - static const unsigned char shift_table[512] = { - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, - 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, - 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13 }; - uint16 hbits = base_table[bits>>23] + static_cast((bits&0x7FFFFF)>>shift_table[bits>>23]); - if(R == std::round_to_nearest) - hbits += (((bits&0x7FFFFF)>>(shift_table[bits>>23]-1))|(((bits>>23)&0xFF)==102)) & ((hbits&0x7C00)!=0x7C00) - #if HALF_ROUND_TIES_TO_EVEN - & (((((static_cast(1)<<(shift_table[bits>>23]-1))-1)&bits)!=0)|hbits) - #endif - ; - else if(R == std::round_toward_zero) - hbits -= ((hbits&0x7FFF)==0x7C00) & ~shift_table[bits>>23]; - else if(R == std::round_toward_infinity) - hbits += ((((bits&0x7FFFFF&((static_cast(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=102)& - ((bits>>23)!=0)))&(hbits<0x7C00)) - ((hbits==0xFC00)&((bits>>23)!=511)); - else if(R == std::round_toward_neg_infinity) - hbits += ((((bits&0x7FFFFF&((static_cast(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=358)& - ((bits>>23)!=256)))&(hbits<0xFC00)&(hbits>>15)) - ((hbits==0x7C00)&((bits>>23)!=255)); - return hbits; - } - - /// Convert IEEE double-precision to half-precision. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \param value double-precision value - /// \return binary representation of half-precision value - template uint16 float2half_impl(double value, true_type) - { - typedef bits::type uint32; - typedef bits::type uint64; - uint64 bits;// = *reinterpret_cast(&value); //violating strict aliasing! - std::memcpy(&bits, &value, sizeof(double)); - uint32 hi = bits >> 32, lo = bits & 0xFFFFFFFF; - uint16 hbits = (hi>>16) & 0x8000; - hi &= 0x7FFFFFFF; - int exp = hi >> 20; - if(exp == 2047) - return hbits | 0x7C00 | (0x3FF&-static_cast((bits&0xFFFFFFFFFFFFF)!=0)); - if(exp > 1038) - { - if(R == std::round_toward_infinity) - return hbits | 0x7C00 - (hbits>>15); - if(R == std::round_toward_neg_infinity) - return hbits | 0x7BFF + (hbits>>15); - return hbits | 0x7BFF + (R!=std::round_toward_zero); - } - int g, s = lo != 0; - if(exp > 1008) - { - g = (hi>>9) & 1; - s |= (hi&0x1FF) != 0; - hbits |= ((exp-1008)<<10) | ((hi>>10)&0x3FF); - } - else if(exp > 997) - { - int i = 1018 - exp; - hi = (hi&0xFFFFF) | 0x100000; - g = (hi>>i) & 1; - s |= (hi&((1L<> (i+1); - } - else - { - g = 0; - s |= hi != 0; - } - if(R == std::round_to_nearest) - #if HALF_ROUND_TIES_TO_EVEN - hbits += g & (s|hbits); - #else - hbits += g; - #endif - else if(R == std::round_toward_infinity) - hbits += ~(hbits>>15) & (s|g); - else if(R == std::round_toward_neg_infinity) - hbits += (hbits>>15) & (g|s); - return hbits; - } - - /// Convert non-IEEE floating point to half-precision. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam T source type (builtin floating point type) - /// \param value floating point value - /// \return binary representation of half-precision value - template uint16 float2half_impl(T value, ...) - { - uint16 hbits = static_cast(builtin_signbit(value)) << 15; - if(value == T()) - return hbits; - if(builtin_isnan(value)) - return hbits | 0x7FFF; - if(builtin_isinf(value)) - return hbits | 0x7C00; - int exp; - std::frexp(value, &exp); - if(exp > 16) - { - if(R == std::round_toward_infinity) - return hbits | (0x7C00 - (hbits>>15)); - else if(R == std::round_toward_neg_infinity) - return hbits | (0x7BFF + (hbits>>15)); - return hbits | (0x7BFF + (R!=std::round_toward_zero)); - } - if(exp < -13) - value = std::ldexp(value, 24); - else - { - value = std::ldexp(value, 11-exp); - hbits |= ((exp+13)<<10); - } - T ival, frac = std::modf(value, &ival); - hbits += static_cast(std::abs(static_cast(ival))); - if(R == std::round_to_nearest) - { - frac = std::abs(frac); - #if HALF_ROUND_TIES_TO_EVEN - hbits += (frac>T(0.5)) | ((frac==T(0.5))&hbits); - #else - hbits += frac >= T(0.5); - #endif - } - else if(R == std::round_toward_infinity) - hbits += frac > T(); - else if(R == std::round_toward_neg_infinity) - hbits += frac < T(); - return hbits; - } - - /// Convert floating point to half-precision. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam T source type (builtin floating point type) - /// \param value floating point value - /// \return binary representation of half-precision value - template uint16 float2half(T value) - { - return float2half_impl(value, bool_type::is_iec559&&sizeof(typename bits::type)==sizeof(T)>()); - } - - /// Convert integer to half-precision floating point. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam S `true` if value negative, `false` else - /// \tparam T type to convert (builtin integer type) - /// \param value non-negative integral value - /// \return binary representation of half-precision value - template uint16 int2half_impl(T value) - { - #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_integral::value, "int to half conversion only supports builtin integer types"); - #endif - if(S) - value = -value; - uint16 bits = S << 15; - if(value > 0xFFFF) - { - if(R == std::round_toward_infinity) - bits |= 0x7C00 - S; - else if(R == std::round_toward_neg_infinity) - bits |= 0x7BFF + S; - else - bits |= 0x7BFF + (R!=std::round_toward_zero); - } - else if(value) - { - unsigned int m = value, exp = 24; - for(; m<0x400; m<<=1,--exp) ; - for(; m>0x7FF; m>>=1,++exp) ; - bits |= (exp<<10) + m; - if(exp > 24) - { - if(R == std::round_to_nearest) - bits += (value>>(exp-25)) & 1 - #if HALF_ROUND_TIES_TO_EVEN - & (((((1<<(exp-25))-1)&value)!=0)|bits) - #endif - ; - else if(R == std::round_toward_infinity) - bits += ((value&((1<<(exp-24))-1))!=0) & !S; - else if(R == std::round_toward_neg_infinity) - bits += ((value&((1<<(exp-24))-1))!=0) & S; - } - } - return bits; - } - - /// Convert integer to half-precision floating point. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam T type to convert (builtin integer type) - /// \param value integral value - /// \return binary representation of half-precision value - template uint16 int2half(T value) - { - return (value<0) ? int2half_impl(value) : int2half_impl(value); - } - - /// Convert half-precision to IEEE single-precision. - /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). - /// \param value binary representation of half-precision value - /// \return single-precision value - inline float half2float_impl(uint16 value, float, true_type) - { - typedef bits::type uint32; -/* uint32 bits = static_cast(value&0x8000) << 16; - int abs = value & 0x7FFF; - if(abs) - { - bits |= 0x38000000 << static_cast(abs>=0x7C00); - for(; abs<0x400; abs<<=1,bits-=0x800000) ; - bits += static_cast(abs) << 13; - } -*/ static const uint32 mantissa_table[2048] = { - 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000, 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000, 0x35700000, - 0x35800000, 0x35880000, 0x35900000, 0x35980000, 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, 0x35F00000, 0x35F80000, - 0x36000000, 0x36040000, 0x36080000, 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000, 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000, - 0x36400000, 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000, 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, 0x36700000, 0x36740000, 0x36780000, 0x367C0000, - 0x36800000, 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000, 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369A0000, 0x369C0000, 0x369E0000, - 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000, - 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000, 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000, - 0x36E00000, 0x36E20000, 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000, 0x36FC0000, 0x36FE0000, - 0x37000000, 0x37010000, 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000, 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000, - 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000, - 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000, 0x372F0000, - 0x37300000, 0x37310000, 0x37320000, 0x37330000, 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, 0x373E0000, 0x373F0000, - 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000, - 0x37500000, 0x37510000, 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000, - 0x37600000, 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000, - 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000, 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000, - 0x37800000, 0x37808000, 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, 0x37870000, 0x37878000, - 0x37880000, 0x37888000, 0x37890000, 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, 0x378F8000, - 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, 0x37960000, 0x37968000, 0x37970000, 0x37978000, - 0x37980000, 0x37988000, 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000, 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000, - 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000, - 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000, 0x37AB0000, 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000, 0x37AF0000, 0x37AF8000, - 0x37B00000, 0x37B08000, 0x37B10000, 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, 0x37B68000, 0x37B70000, 0x37B78000, - 0x37B80000, 0x37B88000, 0x37B90000, 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000, 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000, - 0x37C00000, 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000, - 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, 0x37CF0000, 0x37CF8000, - 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000, 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000, - 0x37D80000, 0x37D88000, 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000, 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000, - 0x37E00000, 0x37E08000, 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000, 0x37E70000, 0x37E78000, - 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000, 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000, - 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000, - 0x37F80000, 0x37F88000, 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, 0x37FF0000, 0x37FF8000, - 0x38000000, 0x38004000, 0x38008000, 0x3800C000, 0x38010000, 0x38014000, 0x38018000, 0x3801C000, 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000, 0x38034000, 0x38038000, 0x3803C000, - 0x38040000, 0x38044000, 0x38048000, 0x3804C000, 0x38050000, 0x38054000, 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, 0x38070000, 0x38074000, 0x38078000, 0x3807C000, - 0x38080000, 0x38084000, 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000, 0x380A0000, 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000, - 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000, 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, 0x380FC000, - 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, 0x38114000, 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000, 0x38130000, 0x38134000, 0x38138000, 0x3813C000, - 0x38140000, 0x38144000, 0x38148000, 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, 0x38174000, 0x38178000, 0x3817C000, - 0x38180000, 0x38184000, 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000, 0x3819C000, 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000, - 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000, 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000, - 0x38200000, 0x38204000, 0x38208000, 0x3820C000, 0x38210000, 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000, 0x38228000, 0x3822C000, 0x38230000, 0x38234000, 0x38238000, 0x3823C000, - 0x38240000, 0x38244000, 0x38248000, 0x3824C000, 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000, - 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000, 0x38298000, 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000, 0x382B4000, 0x382B8000, 0x382BC000, - 0x382C0000, 0x382C4000, 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000, - 0x38300000, 0x38304000, 0x38308000, 0x3830C000, 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000, 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000, - 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000, 0x38358000, 0x3835C000, 0x38360000, 0x38364000, 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000, 0x3837C000, - 0x38380000, 0x38384000, 0x38388000, 0x3838C000, 0x38390000, 0x38394000, 0x38398000, 0x3839C000, 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, 0x383B8000, 0x383BC000, - 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000, 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000, 0x383E8000, 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000, - 0x38400000, 0x38404000, 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000, 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, 0x38430000, 0x38434000, 0x38438000, 0x3843C000, - 0x38440000, 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, 0x3845C000, 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000, 0x38478000, 0x3847C000, - 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000, 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000, - 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000, - 0x38500000, 0x38504000, 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000, 0x38538000, 0x3853C000, - 0x38540000, 0x38544000, 0x38548000, 0x3854C000, 0x38550000, 0x38554000, 0x38558000, 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000, 0x38570000, 0x38574000, 0x38578000, 0x3857C000, - 0x38580000, 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000, - 0x385C0000, 0x385C4000, 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000, 0x385FC000, - 0x38600000, 0x38604000, 0x38608000, 0x3860C000, 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000, 0x38638000, 0x3863C000, - 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, 0x38654000, 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000, 0x38670000, 0x38674000, 0x38678000, 0x3867C000, - 0x38680000, 0x38684000, 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000, 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000, - 0x386C0000, 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000, - 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000, 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000, 0x38730000, 0x38734000, 0x38738000, 0x3873C000, - 0x38740000, 0x38744000, 0x38748000, 0x3874C000, 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000, 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, 0x38778000, 0x3877C000, - 0x38780000, 0x38784000, 0x38788000, 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000, - 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000, 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000, - 0x38000000, 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800A000, 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000, - 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000, - 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, 0x38050000, 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000, 0x3805C000, 0x3805E000, - 0x38060000, 0x38062000, 0x38064000, 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807A000, 0x3807C000, 0x3807E000, - 0x38080000, 0x38082000, 0x38084000, 0x38086000, 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000, 0x38094000, 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000, - 0x380A0000, 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000, - 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, 0x380CE000, 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, 0x380DC000, 0x380DE000, - 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000, 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000, - 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000, - 0x38120000, 0x38122000, 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000, 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000, 0x3813C000, 0x3813E000, - 0x38140000, 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000, 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000, 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000, - 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, 0x38176000, 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000, - 0x38180000, 0x38182000, 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, 0x3819C000, 0x3819E000, - 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, 0x381BC000, 0x381BE000, - 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000, - 0x381E0000, 0x381E2000, 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000, - 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, 0x3821E000, - 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000, 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823A000, 0x3823C000, 0x3823E000, - 0x38240000, 0x38242000, 0x38244000, 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825A000, 0x3825C000, 0x3825E000, - 0x38260000, 0x38262000, 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, 0x38272000, 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000, - 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000, - 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000, 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, 0x382BC000, 0x382BE000, - 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000, 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000, - 0x382E0000, 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000, 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000, 0x382FA000, 0x382FC000, 0x382FE000, - 0x38300000, 0x38302000, 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831A000, 0x3831C000, 0x3831E000, - 0x38320000, 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000, 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000, - 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000, - 0x38360000, 0x38362000, 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, 0x3837C000, 0x3837E000, - 0x38380000, 0x38382000, 0x38384000, 0x38386000, 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000, - 0x383A0000, 0x383A2000, 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000, - 0x383C0000, 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000, 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, 0x383DE000, - 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000, 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, 0x383FE000, - 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000, - 0x38420000, 0x38422000, 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000, 0x3843C000, 0x3843E000, - 0x38440000, 0x38442000, 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000, 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000, - 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000, - 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000, 0x3849E000, - 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, 0x384BE000, - 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000, 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000, 0x384DA000, 0x384DC000, 0x384DE000, - 0x384E0000, 0x384E2000, 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000, - 0x38500000, 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000, - 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000, 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, 0x38538000, 0x3853A000, 0x3853C000, 0x3853E000, - 0x38540000, 0x38542000, 0x38544000, 0x38546000, 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, 0x3855C000, 0x3855E000, - 0x38560000, 0x38562000, 0x38564000, 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000, - 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000, 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000, - 0x385A0000, 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, 0x385BA000, 0x385BC000, 0x385BE000, - 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000, - 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000, 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000, - 0x38600000, 0x38602000, 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000, 0x3861C000, 0x3861E000, - 0x38620000, 0x38622000, 0x38624000, 0x38626000, 0x38628000, 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000, - 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000, - 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000, 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000, 0x3867E000, - 0x38680000, 0x38682000, 0x38684000, 0x38686000, 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869A000, 0x3869C000, 0x3869E000, - 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000, - 0x386C0000, 0x386C2000, 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000, 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000, - 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000, 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000, - 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000, - 0x38720000, 0x38722000, 0x38724000, 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, 0x3873C000, 0x3873E000, - 0x38740000, 0x38742000, 0x38744000, 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, 0x3875E000, - 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000, 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000, - 0x38780000, 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000, - 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000, - 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000, 0x387CC000, 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000, 0x387DC000, 0x387DE000, - 0x387E0000, 0x387E2000, 0x387E4000, 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, 0x387FA000, 0x387FC000, 0x387FE000 }; - static const uint32 exponent_table[64] = { - 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, 0x06000000, 0x06800000, 0x07000000, 0x07800000, - 0x08000000, 0x08800000, 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, 0x0F000000, 0x47800000, - 0x80000000, 0x80800000, 0x81000000, 0x81800000, 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000, - 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000 }; - static const unsigned short offset_table[64] = { - 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, - 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024 }; - uint32 bits = mantissa_table[offset_table[value>>10]+(value&0x3FF)] + exponent_table[value>>10]; -// return *reinterpret_cast(&bits); //violating strict aliasing! - float out; - std::memcpy(&out, &bits, sizeof(float)); - return out; - } - - /// Convert half-precision to IEEE double-precision. - /// \param value binary representation of half-precision value - /// \return double-precision value - inline double half2float_impl(uint16 value, double, true_type) - { - typedef bits::type uint32; - typedef bits::type uint64; - uint32 hi = static_cast(value&0x8000) << 16; - int abs = value & 0x7FFF; - if(abs) - { - hi |= 0x3F000000 << static_cast(abs>=0x7C00); - for(; abs<0x400; abs<<=1,hi-=0x100000) ; - hi += static_cast(abs) << 10; - } - uint64 bits = static_cast(hi) << 32; -// return *reinterpret_cast(&bits); //violating strict aliasing! - double out; - std::memcpy(&out, &bits, sizeof(double)); - return out; - } - - /// Convert half-precision to non-IEEE floating point. - /// \tparam T type to convert to (builtin integer type) - /// \param value binary representation of half-precision value - /// \return floating point value - template T half2float_impl(uint16 value, T, ...) - { - T out; - int abs = value & 0x7FFF; - if(abs > 0x7C00) - out = std::numeric_limits::has_quiet_NaN ? std::numeric_limits::quiet_NaN() : T(); - else if(abs == 0x7C00) - out = std::numeric_limits::has_infinity ? std::numeric_limits::infinity() : std::numeric_limits::max(); - else if(abs > 0x3FF) - out = std::ldexp(static_cast((abs&0x3FF)|0x400), (abs>>10)-25); - else - out = std::ldexp(static_cast(abs), -24); - return (value&0x8000) ? -out : out; - } - - /// Convert half-precision to floating point. - /// \tparam T type to convert to (builtin integer type) - /// \param value binary representation of half-precision value - /// \return floating point value - template T half2float(uint16 value) - { - return half2float_impl(value, T(), bool_type::is_iec559&&sizeof(typename bits::type)==sizeof(T)>()); - } - - /// Convert half-precision floating point to integer. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam E `true` for round to even, `false` for round away from zero - /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) - /// \param value binary representation of half-precision value - /// \return integral value - template T half2int_impl(uint16 value) - { - #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_integral::value, "half to int conversion only supports builtin integer types"); - #endif - unsigned int e = value & 0x7FFF; - if(e >= 0x7C00) - return (value&0x8000) ? std::numeric_limits::min() : std::numeric_limits::max(); - if(e < 0x3800) - { - if(R == std::round_toward_infinity) - return T(~(value>>15)&(e!=0)); - else if(R == std::round_toward_neg_infinity) - return -T(value>0x8000); - return T(); - } - unsigned int m = (value&0x3FF) | 0x400; - e >>= 10; - if(e < 25) - { - if(R == std::round_to_nearest) - m += (1<<(24-e)) - (~(m>>(25-e))&E); - else if(R == std::round_toward_infinity) - m += ((value>>15)-1) & ((1<<(25-e))-1U); - else if(R == std::round_toward_neg_infinity) - m += -(value>>15) & ((1<<(25-e))-1U); - m >>= 25 - e; - } - else - m <<= e - 25; - return (value&0x8000) ? -static_cast(m) : static_cast(m); - } - - /// Convert half-precision floating point to integer. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) - /// \param value binary representation of half-precision value - /// \return integral value - template T half2int(uint16 value) { return half2int_impl(value); } - - /// Convert half-precision floating point to integer using round-to-nearest-away-from-zero. - /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) - /// \param value binary representation of half-precision value - /// \return integral value - template T half2int_up(uint16 value) { return half2int_impl(value); } - - /// Round half-precision number to nearest integer value. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam E `true` for round to even, `false` for round away from zero - /// \param value binary representation of half-precision value - /// \return half-precision bits for nearest integral value - template uint16 round_half_impl(uint16 value) - { - unsigned int e = value & 0x7FFF; - uint16 result = value; - if(e < 0x3C00) - { - result &= 0x8000; - if(R == std::round_to_nearest) - result |= 0x3C00U & -(e>=(0x3800+E)); - else if(R == std::round_toward_infinity) - result |= 0x3C00U & -(~(value>>15)&(e!=0)); - else if(R == std::round_toward_neg_infinity) - result |= 0x3C00U & -(value>0x8000); - } - else if(e < 0x6400) - { - e = 25 - (e>>10); - unsigned int mask = (1<>e)&E); - else if(R == std::round_toward_infinity) - result += mask & ((value>>15)-1); - else if(R == std::round_toward_neg_infinity) - result += mask & -(value>>15); - result &= ~mask; - } - return result; - } - - /// Round half-precision number to nearest integer value. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \param value binary representation of half-precision value - /// \return half-precision bits for nearest integral value - template uint16 round_half(uint16 value) { return round_half_impl(value); } - - /// Round half-precision number to nearest integer value using round-to-nearest-away-from-zero. - /// \param value binary representation of half-precision value - /// \return half-precision bits for nearest integral value - inline uint16 round_half_up(uint16 value) { return round_half_impl(value); } - /// \} - - struct functions; - template struct unary_specialized; - template struct binary_specialized; - template struct half_caster; - } - - /// Half-precision floating point type. - /// This class implements an IEEE-conformant half-precision floating point type with the usual arithmetic operators and - /// conversions. It is implicitly convertible to single-precision floating point, which makes artihmetic expressions and - /// functions with mixed-type operands to be of the most precise operand type. Additionally all arithmetic operations - /// (and many mathematical functions) are carried out in single-precision internally. All conversions from single- to - /// half-precision are done using the library's default rounding mode, but temporary results inside chained arithmetic - /// expressions are kept in single-precision as long as possible (while of course still maintaining a strong half-precision type). - /// - /// According to the C++98/03 definition, the half type is not a POD type. But according to C++11's less strict and - /// extended definitions it is both a standard layout type and a trivially copyable type (even if not a POD type), which - /// means it can be standard-conformantly copied using raw binary copies. But in this context some more words about the - /// actual size of the type. Although the half is representing an IEEE 16-bit type, it does not neccessarily have to be of - /// exactly 16-bits size. But on any reasonable implementation the actual binary representation of this type will most - /// probably not ivolve any additional "magic" or padding beyond the simple binary representation of the underlying 16-bit - /// IEEE number, even if not strictly guaranteed by the standard. But even then it only has an actual size of 16 bits if - /// your C++ implementation supports an unsigned integer type of exactly 16 bits width. But this should be the case on - /// nearly any reasonable platform. - /// - /// So if your C++ implementation is not totally exotic or imposes special alignment requirements, it is a reasonable - /// assumption that the data of a half is just comprised of the 2 bytes of the underlying IEEE representation. - class half - { - friend struct detail::functions; - friend struct detail::unary_specialized; - friend struct detail::binary_specialized; - template friend struct detail::half_caster; - friend class std::numeric_limits; - #if HALF_ENABLE_CPP11_HASH - friend struct std::hash; - #endif - #if HALF_ENABLE_CPP11_USER_LITERALS - friend half literal::operator"" _h(long double); - #endif - - public: - /// Default constructor. - /// This initializes the half to 0. Although this does not match the builtin types' default-initialization semantics - /// and may be less efficient than no initialization, it is needed to provide proper value-initialization semantics. - HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {} - - /// Copy constructor. - /// \tparam T type of concrete half expression - /// \param rhs half expression to copy from - half(detail::expr rhs) : data_(detail::float2half(static_cast(rhs))) {} - - /// Conversion constructor. - /// \param rhs float to convert - explicit half(float rhs) : data_(detail::float2half(rhs)) {} - - /// Conversion to single-precision. - /// \return single precision value representing expression value - operator float() const { return detail::half2float(data_); } - - /// Assignment operator. - /// \tparam T type of concrete half expression - /// \param rhs half expression to copy from - /// \return reference to this half - half& operator=(detail::expr rhs) { return *this = static_cast(rhs); } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to add - /// \return reference to this half - template typename detail::enable::type operator+=(T rhs) { return *this += static_cast(rhs); } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to subtract - /// \return reference to this half - template typename detail::enable::type operator-=(T rhs) { return *this -= static_cast(rhs); } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to multiply with - /// \return reference to this half - template typename detail::enable::type operator*=(T rhs) { return *this *= static_cast(rhs); } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to divide by - /// \return reference to this half - template typename detail::enable::type operator/=(T rhs) { return *this /= static_cast(rhs); } - - /// Assignment operator. - /// \param rhs single-precision value to copy from - /// \return reference to this half - half& operator=(float rhs) { data_ = detail::float2half(rhs); return *this; } - - /// Arithmetic assignment. - /// \param rhs single-precision value to add - /// \return reference to this half - half& operator+=(float rhs) { data_ = detail::float2half(detail::half2float(data_)+rhs); return *this; } - - /// Arithmetic assignment. - /// \param rhs single-precision value to subtract - /// \return reference to this half - half& operator-=(float rhs) { data_ = detail::float2half(detail::half2float(data_)-rhs); return *this; } - - /// Arithmetic assignment. - /// \param rhs single-precision value to multiply with - /// \return reference to this half - half& operator*=(float rhs) { data_ = detail::float2half(detail::half2float(data_)*rhs); return *this; } - - /// Arithmetic assignment. - /// \param rhs single-precision value to divide by - /// \return reference to this half - half& operator/=(float rhs) { data_ = detail::float2half(detail::half2float(data_)/rhs); return *this; } - - /// Prefix increment. - /// \return incremented half value - half& operator++() { return *this += 1.0f; } - - /// Prefix decrement. - /// \return decremented half value - half& operator--() { return *this -= 1.0f; } - - /// Postfix increment. - /// \return non-incremented half value - half operator++(int) { half out(*this); ++*this; return out; } - - /// Postfix decrement. - /// \return non-decremented half value - half operator--(int) { half out(*this); --*this; return out; } - - private: - /// Rounding mode to use - static const std::float_round_style round_style = (std::float_round_style)(HALF_ROUND_STYLE); - - /// Constructor. - /// \param bits binary representation to set half to - HALF_CONSTEXPR half(detail::binary_t, detail::uint16 bits) HALF_NOEXCEPT : data_(bits) {} - - /// Internal binary representation - detail::uint16 data_; - }; - -#if HALF_ENABLE_CPP11_USER_LITERALS - namespace literal - { - /// Half literal. - /// While this returns an actual half-precision value, half literals can unfortunately not be constant expressions due - /// to rather involved conversions. - /// \param value literal value - /// \return half with given value (if representable) - inline half operator"" _h(long double value) { return half(detail::binary, detail::float2half(value)); } - } -#endif - - namespace detail - { - /// Wrapper implementing unspecialized half-precision functions. - struct functions - { - /// Addition implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision sum stored in single-precision - static expr plus(float x, float y) { return expr(x+y); } - - /// Subtraction implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision difference stored in single-precision - static expr minus(float x, float y) { return expr(x-y); } - - /// Multiplication implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision product stored in single-precision - static expr multiplies(float x, float y) { return expr(x*y); } - - /// Division implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision quotient stored in single-precision - static expr divides(float x, float y) { return expr(x/y); } - - /// Output implementation. - /// \param out stream to write to - /// \param arg value to write - /// \return reference to stream - template static std::basic_ostream& write(std::basic_ostream &out, float arg) { return out << arg; } - - /// Input implementation. - /// \param in stream to read from - /// \param arg half to read into - /// \return reference to stream - template static std::basic_istream& read(std::basic_istream &in, half &arg) - { - float f; - if(in >> f) - arg = f; - return in; - } - - /// Modulo implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision division remainder stored in single-precision - static expr fmod(float x, float y) { return expr(std::fmod(x, y)); } - - /// Remainder implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision division remainder stored in single-precision - static expr remainder(float x, float y) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::remainder(x, y)); - #else - if(builtin_isnan(x) || builtin_isnan(y)) - return expr(std::numeric_limits::quiet_NaN()); - float ax = std::fabs(x), ay = std::fabs(y); - if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) - return expr(std::numeric_limits::quiet_NaN()); - if(ay >= 65536.0f) - return expr(x); - if(ax == ay) - return expr(builtin_signbit(x) ? -0.0f : 0.0f); - ax = std::fmod(ax, ay+ay); - float y2 = 0.5f * ay; - if(ax > y2) - { - ax -= ay; - if(ax >= y2) - ax -= ay; - } - return expr(builtin_signbit(x) ? -ax : ax); - #endif - } - - /// Remainder implementation. - /// \param x first operand - /// \param y second operand - /// \param quo address to store quotient bits at - /// \return Half-precision division remainder stored in single-precision - static expr remquo(float x, float y, int *quo) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::remquo(x, y, quo)); - #else - if(builtin_isnan(x) || builtin_isnan(y)) - return expr(std::numeric_limits::quiet_NaN()); - bool sign = builtin_signbit(x), qsign = static_cast(sign^builtin_signbit(y)); - float ax = std::fabs(x), ay = std::fabs(y); - if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) - return expr(std::numeric_limits::quiet_NaN()); - if(ay >= 65536.0f) - return expr(x); - if(ax == ay) - return *quo = qsign ? -1 : 1, expr(sign ? -0.0f : 0.0f); - ax = std::fmod(ax, 8.0f*ay); - int cquo = 0; - if(ax >= 4.0f * ay) - { - ax -= 4.0f * ay; - cquo += 4; - } - if(ax >= 2.0f * ay) - { - ax -= 2.0f * ay; - cquo += 2; - } - float y2 = 0.5f * ay; - if(ax > y2) - { - ax -= ay; - ++cquo; - if(ax >= y2) - { - ax -= ay; - ++cquo; - } - } - return *quo = qsign ? -cquo : cquo, expr(sign ? -ax : ax); - #endif - } - - /// Positive difference implementation. - /// \param x first operand - /// \param y second operand - /// \return Positive difference stored in single-precision - static expr fdim(float x, float y) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::fdim(x, y)); - #else - return expr((x<=y) ? 0.0f : (x-y)); - #endif - } - - /// Fused multiply-add implementation. - /// \param x first operand - /// \param y second operand - /// \param z third operand - /// \return \a x * \a y + \a z stored in single-precision - static expr fma(float x, float y, float z) - { - #if HALF_ENABLE_CPP11_CMATH && defined(FP_FAST_FMAF) - return expr(std::fma(x, y, z)); - #else - return expr(x*y+z); - #endif - } - - /// Get NaN. - /// \return Half-precision quiet NaN - static half nanh() { return half(binary, 0x7FFF); } - - /// Exponential implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr exp(float arg) { return expr(std::exp(arg)); } - - /// Exponential implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr expm1(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::expm1(arg)); - #else - return expr(static_cast(std::exp(static_cast(arg))-1.0)); - #endif - } - - /// Binary exponential implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr exp2(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::exp2(arg)); - #else - return expr(static_cast(std::exp(arg*0.69314718055994530941723212145818))); - #endif - } - - /// Logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr log(float arg) { return expr(std::log(arg)); } - - /// Common logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr log10(float arg) { return expr(std::log10(arg)); } - - /// Logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr log1p(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::log1p(arg)); - #else - return expr(static_cast(std::log(1.0+arg))); - #endif - } - - /// Binary logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr log2(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::log2(arg)); - #else - return expr(static_cast(std::log(static_cast(arg))*1.4426950408889634073599246810019)); - #endif - } - - /// Square root implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr sqrt(float arg) { return expr(std::sqrt(arg)); } - - /// Cubic root implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr cbrt(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::cbrt(arg)); - #else - if(builtin_isnan(arg) || builtin_isinf(arg)) - return expr(arg); - return expr(builtin_signbit(arg) ? -static_cast(std::pow(-static_cast(arg), 1.0/3.0)) : - static_cast(std::pow(static_cast(arg), 1.0/3.0))); - #endif - } - - /// Hypotenuse implementation. - /// \param x first argument - /// \param y second argument - /// \return function value stored in single-preicision - static expr hypot(float x, float y) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::hypot(x, y)); - #else - return expr((builtin_isinf(x) || builtin_isinf(y)) ? std::numeric_limits::infinity() : - static_cast(std::sqrt(static_cast(x)*x+static_cast(y)*y))); - #endif - } +*/ static const uint16 + base_table[512] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, + 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100, + 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00, + 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00, + 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, 0x5C00, + 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, + 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100, + 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, + 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00, + 0xC000, 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, + 0xE000, 0xE400, 0xE800, 0xEC00, 0xF000, 0xF400, 0xF800, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00}; + static const unsigned char shift_table[512] = { + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, + 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 23, + 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 13}; + uint16 hbits = + base_table[bits >> 23] + + static_cast((bits & 0x7FFFFF) >> shift_table[bits >> 23]); + if (R == std::round_to_nearest) + hbits += + (((bits & 0x7FFFFF) >> (shift_table[bits >> 23] - 1)) | + (((bits >> 23) & 0xFF) == 102)) & + ((hbits & 0x7C00) != 0x7C00) +#if HALF_ROUND_TIES_TO_EVEN + & (((((static_cast(1) << (shift_table[bits >> 23] - 1)) - 1) & + bits) != 0) | + hbits) +#endif + ; + else if (R == std::round_toward_zero) + hbits -= ((hbits & 0x7FFF) == 0x7C00) & ~shift_table[bits >> 23]; + else if (R == std::round_toward_infinity) + hbits += + ((((bits & 0x7FFFFF & + ((static_cast(1) << (shift_table[bits >> 23])) - 1)) != 0) | + (((bits >> 23) <= 102) & ((bits >> 23) != 0))) & + (hbits < 0x7C00)) - + ((hbits == 0xFC00) & ((bits >> 23) != 511)); + else if (R == std::round_toward_neg_infinity) + hbits += + ((((bits & 0x7FFFFF & + ((static_cast(1) << (shift_table[bits >> 23])) - 1)) != 0) | + (((bits >> 23) <= 358) & ((bits >> 23) != 256))) & + (hbits < 0xFC00) & (hbits >> 15)) - + ((hbits == 0x7C00) & ((bits >> 23) != 255)); + return hbits; +} - /// Power implementation. - /// \param base value to exponentiate - /// \param exp power to expontiate to - /// \return function value stored in single-preicision - static expr pow(float base, float exp) { return expr(std::pow(base, exp)); } - - /// Sine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr sin(float arg) { return expr(std::sin(arg)); } - - /// Cosine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr cos(float arg) { return expr(std::cos(arg)); } - - /// Tan implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr tan(float arg) { return expr(std::tan(arg)); } - - /// Arc sine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr asin(float arg) { return expr(std::asin(arg)); } - - /// Arc cosine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr acos(float arg) { return expr(std::acos(arg)); } - - /// Arc tangent implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr atan(float arg) { return expr(std::atan(arg)); } - - /// Arc tangent implementation. - /// \param x first argument - /// \param y second argument - /// \return function value stored in single-preicision - static expr atan2(float x, float y) { return expr(std::atan2(x, y)); } - - /// Hyperbolic sine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr sinh(float arg) { return expr(std::sinh(arg)); } - - /// Hyperbolic cosine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr cosh(float arg) { return expr(std::cosh(arg)); } - - /// Hyperbolic tangent implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr tanh(float arg) { return expr(std::tanh(arg)); } - - /// Hyperbolic area sine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr asinh(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::asinh(arg)); - #else - return expr((arg==-std::numeric_limits::infinity()) ? arg : static_cast(std::log(arg+std::sqrt(arg*arg+1.0)))); - #endif - } +/// Convert IEEE double-precision to half-precision. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding +/// \param value double-precision value +/// \return binary representation of half-precision value +template +uint16 float2half_impl(double value, true_type) { + typedef bits::type uint32; + typedef bits::type uint64; + uint64 + bits; // = *reinterpret_cast(&value); //violating + // strict aliasing! + std::memcpy(&bits, &value, sizeof(double)); + uint32 hi = bits >> 32, lo = bits & 0xFFFFFFFF; + uint16 hbits = (hi >> 16) & 0x8000; + hi &= 0x7FFFFFFF; + int exp = hi >> 20; + if (exp == 2047) + return hbits | 0x7C00 | + (0x3FF & -static_cast((bits & 0xFFFFFFFFFFFFF) != 0)); + if (exp > 1038) { + if (R == std::round_toward_infinity) return hbits | 0x7C00 - (hbits >> 15); + if (R == std::round_toward_neg_infinity) + return hbits | 0x7BFF + (hbits >> 15); + return hbits | 0x7BFF + (R != std::round_toward_zero); + } + int g, s = lo != 0; + if (exp > 1008) { + g = (hi >> 9) & 1; + s |= (hi & 0x1FF) != 0; + hbits |= ((exp - 1008) << 10) | ((hi >> 10) & 0x3FF); + } else if (exp > 997) { + int i = 1018 - exp; + hi = (hi & 0xFFFFF) | 0x100000; + g = (hi >> i) & 1; + s |= (hi & ((1L << i) - 1)) != 0; + hbits |= hi >> (i + 1); + } else { + g = 0; + s |= hi != 0; + } + if (R == std::round_to_nearest) +#if HALF_ROUND_TIES_TO_EVEN + hbits += g & (s | hbits); +#else + hbits += g; +#endif + else if (R == std::round_toward_infinity) + hbits += ~(hbits >> 15) & (s | g); + else if (R == std::round_toward_neg_infinity) + hbits += (hbits >> 15) & (g | s); + return hbits; +} - /// Hyperbolic area cosine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr acosh(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::acosh(arg)); - #else - return expr((arg<-1.0f) ? std::numeric_limits::quiet_NaN() : static_cast(std::log(arg+std::sqrt(arg*arg-1.0)))); - #endif - } +/// Convert non-IEEE floating point to half-precision. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding +/// \tparam T source type (builtin floating point type) +/// \param value floating point value +/// \return binary representation of half-precision value +template +uint16 float2half_impl(T value, ...) { + uint16 hbits = static_cast(builtin_signbit(value)) << 15; + if (value == T()) return hbits; + if (builtin_isnan(value)) return hbits | 0x7FFF; + if (builtin_isinf(value)) return hbits | 0x7C00; + int exp; + std::frexp(value, &exp); + if (exp > 16) { + if (R == std::round_toward_infinity) + return hbits | (0x7C00 - (hbits >> 15)); + else if (R == std::round_toward_neg_infinity) + return hbits | (0x7BFF + (hbits >> 15)); + return hbits | (0x7BFF + (R != std::round_toward_zero)); + } + if (exp < -13) + value = std::ldexp(value, 24); + else { + value = std::ldexp(value, 11 - exp); + hbits |= ((exp + 13) << 10); + } + T ival, frac = std::modf(value, &ival); + hbits += static_cast(std::abs(static_cast(ival))); + if (R == std::round_to_nearest) { + frac = std::abs(frac); +#if HALF_ROUND_TIES_TO_EVEN + hbits += (frac > T(0.5)) | ((frac == T(0.5)) & hbits); +#else + hbits += frac >= T(0.5); +#endif + } else if (R == std::round_toward_infinity) + hbits += frac > T(); + else if (R == std::round_toward_neg_infinity) + hbits += frac < T(); + return hbits; +} - /// Hyperbolic area tangent implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr atanh(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::atanh(arg)); - #else - return expr(static_cast(0.5*std::log((1.0+arg)/(1.0-arg)))); - #endif - } +/// Convert floating point to half-precision. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding +/// \tparam T source type (builtin floating point type) +/// \param value floating point value +/// \return binary representation of half-precision value +template +uint16 float2half(T value) { + return float2half_impl( + value, bool_type < std::numeric_limits::is_iec559 && + sizeof(typename bits::type) == sizeof(T) > ()); +} - /// Error function implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr erf(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::erf(arg)); - #else - return expr(static_cast(erf(static_cast(arg)))); - #endif - } +/// Convert integer to half-precision floating point. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding +/// \tparam S `true` if value negative, `false` else +/// \tparam T type to convert (builtin integer type) +/// \param value non-negative integral value +/// \return binary representation of half-precision value +template +uint16 int2half_impl(T value) { +#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_integral::value, + "int to half conversion only supports builtin integer types"); +#endif + if (S) value = -value; + uint16 bits = S << 15; + if (value > 0xFFFF) { + if (R == std::round_toward_infinity) + bits |= 0x7C00 - S; + else if (R == std::round_toward_neg_infinity) + bits |= 0x7BFF + S; + else + bits |= 0x7BFF + (R != std::round_toward_zero); + } else if (value) { + unsigned int m = value, exp = 24; + for (; m < 0x400; m <<= 1, --exp) + ; + for (; m > 0x7FF; m >>= 1, ++exp) + ; + bits |= (exp << 10) + m; + if (exp > 24) { + if (R == std::round_to_nearest) + bits += (value >> (exp - 25)) & 1 +#if HALF_ROUND_TIES_TO_EVEN + & (((((1 << (exp - 25)) - 1) & value) != 0) | bits) +#endif + ; + else if (R == std::round_toward_infinity) + bits += ((value & ((1 << (exp - 24)) - 1)) != 0) & !S; + else if (R == std::round_toward_neg_infinity) + bits += ((value & ((1 << (exp - 24)) - 1)) != 0) & S; + } + } + return bits; +} - /// Complementary implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr erfc(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::erfc(arg)); - #else - return expr(static_cast(1.0-erf(static_cast(arg)))); - #endif - } +/// Convert integer to half-precision floating point. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding +/// \tparam T type to convert (builtin integer type) +/// \param value integral value +/// \return binary representation of half-precision value +template +uint16 int2half(T value) { + return (value < 0) ? int2half_impl(value) + : int2half_impl(value); +} - /// Gamma logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr lgamma(float arg) +/// Convert half-precision to IEEE single-precision. +/// Credit for this goes to [Jeroen van der +/// Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). +/// \param value binary representation of half-precision value +/// \return single-precision value +inline float half2float_impl(uint16 value, float, true_type) { + typedef bits::type uint32; + /* uint32 bits = static_cast(value&0x8000) << 16; + int abs = value & 0x7FFF; + if(abs) { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::lgamma(arg)); - #else - if(builtin_isinf(arg)) - return expr(std::numeric_limits::infinity()); - if(arg < 0.0f) - { - float i, f = std::modf(-arg, &i); - if(f == 0.0f) - return expr(std::numeric_limits::infinity()); - return expr(static_cast(1.1447298858494001741434273513531- - std::log(std::abs(std::sin(3.1415926535897932384626433832795*f)))-lgamma(1.0-arg))); - } - return expr(static_cast(lgamma(static_cast(arg)))); - #endif + bits |= 0x38000000 << static_cast(abs>=0x7C00); + for(; abs<0x400; abs<<=1,bits-=0x800000) ; + bits += static_cast(abs) << 13; } +*/ static const uint32 + mantissa_table[2048] = { + 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, + 0x34A00000, 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, + 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000, + 0x35700000, 0x35800000, 0x35880000, 0x35900000, 0x35980000, + 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000, + 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, + 0x35F00000, 0x35F80000, 0x36000000, 0x36040000, 0x36080000, + 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000, + 0x36200000, 0x36240000, 0x36280000, 0x362C0000, 0x36300000, + 0x36340000, 0x36380000, 0x363C0000, 0x36400000, 0x36440000, + 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000, + 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, + 0x36700000, 0x36740000, 0x36780000, 0x367C0000, 0x36800000, + 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000, + 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000, + 0x36960000, 0x36980000, 0x369A0000, 0x369C0000, 0x369E0000, + 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, 0x36A80000, + 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, + 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, + 0x36BE0000, 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, + 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000, + 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000, + 0x36DC0000, 0x36DE0000, 0x36E00000, 0x36E20000, 0x36E40000, + 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, + 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, + 0x36FA0000, 0x36FC0000, 0x36FE0000, 0x37000000, 0x37010000, + 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000, + 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000, + 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000, 0x37100000, + 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, + 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, + 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000, + 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000, + 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000, + 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000, + 0x372F0000, 0x37300000, 0x37310000, 0x37320000, 0x37330000, + 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, + 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, + 0x373E0000, 0x373F0000, 0x37400000, 0x37410000, 0x37420000, + 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000, + 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, 0x374C0000, + 0x374D0000, 0x374E0000, 0x374F0000, 0x37500000, 0x37510000, + 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, + 0x37570000, 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, + 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000, 0x37600000, + 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, + 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376A0000, + 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000, + 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, + 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, + 0x377A0000, 0x377B0000, 0x377C0000, 0x377D0000, 0x377E0000, + 0x377F0000, 0x37800000, 0x37808000, 0x37810000, 0x37818000, + 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, + 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, + 0x37870000, 0x37878000, 0x37880000, 0x37888000, 0x37890000, + 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, + 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000, + 0x378E8000, 0x378F0000, 0x378F8000, 0x37900000, 0x37908000, + 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000, + 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, + 0x37960000, 0x37968000, 0x37970000, 0x37978000, 0x37980000, + 0x37988000, 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, + 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000, + 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000, + 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, 0x37A20000, + 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, + 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, + 0x37A78000, 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, + 0x37AA0000, 0x37AA8000, 0x37AB0000, 0x37AB8000, 0x37AC0000, + 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000, + 0x37AF0000, 0x37AF8000, 0x37B00000, 0x37B08000, 0x37B10000, + 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, + 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, + 0x37B68000, 0x37B70000, 0x37B78000, 0x37B80000, 0x37B88000, + 0x37B90000, 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000, + 0x37BB8000, 0x37BC0000, 0x37BC8000, 0x37BD0000, 0x37BD8000, + 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000, 0x37C00000, + 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, + 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, + 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000, + 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000, + 0x37CA8000, 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000, + 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, 0x37CF0000, + 0x37CF8000, 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, + 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, + 0x37D48000, 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, + 0x37D70000, 0x37D78000, 0x37D80000, 0x37D88000, 0x37D90000, + 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000, + 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000, + 0x37DE8000, 0x37DF0000, 0x37DF8000, 0x37E00000, 0x37E08000, + 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, + 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, + 0x37E60000, 0x37E68000, 0x37E70000, 0x37E78000, 0x37E80000, + 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, 0x37EA8000, + 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000, + 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000, + 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, + 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, + 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000, + 0x37F78000, 0x37F80000, 0x37F88000, 0x37F90000, 0x37F98000, + 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, 0x37FC0000, + 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, + 0x37FF0000, 0x37FF8000, 0x38000000, 0x38004000, 0x38008000, + 0x3800C000, 0x38010000, 0x38014000, 0x38018000, 0x3801C000, + 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000, + 0x38034000, 0x38038000, 0x3803C000, 0x38040000, 0x38044000, + 0x38048000, 0x3804C000, 0x38050000, 0x38054000, 0x38058000, + 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, + 0x38070000, 0x38074000, 0x38078000, 0x3807C000, 0x38080000, + 0x38084000, 0x38088000, 0x3808C000, 0x38090000, 0x38094000, + 0x38098000, 0x3809C000, 0x380A0000, 0x380A4000, 0x380A8000, + 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000, + 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000, + 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, + 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, + 0x380FC000, 0x38100000, 0x38104000, 0x38108000, 0x3810C000, + 0x38110000, 0x38114000, 0x38118000, 0x3811C000, 0x38120000, + 0x38124000, 0x38128000, 0x3812C000, 0x38130000, 0x38134000, + 0x38138000, 0x3813C000, 0x38140000, 0x38144000, 0x38148000, + 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, + 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, + 0x38174000, 0x38178000, 0x3817C000, 0x38180000, 0x38184000, + 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000, + 0x3819C000, 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000, + 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000, 0x381C0000, + 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, + 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, + 0x381EC000, 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000, + 0x38200000, 0x38204000, 0x38208000, 0x3820C000, 0x38210000, + 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000, + 0x38228000, 0x3822C000, 0x38230000, 0x38234000, 0x38238000, + 0x3823C000, 0x38240000, 0x38244000, 0x38248000, 0x3824C000, + 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, + 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000, + 0x38278000, 0x3827C000, 0x38280000, 0x38284000, 0x38288000, + 0x3828C000, 0x38290000, 0x38294000, 0x38298000, 0x3829C000, + 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000, + 0x382B4000, 0x382B8000, 0x382BC000, 0x382C0000, 0x382C4000, + 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, + 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, + 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000, 0x38300000, + 0x38304000, 0x38308000, 0x3830C000, 0x38310000, 0x38314000, + 0x38318000, 0x3831C000, 0x38320000, 0x38324000, 0x38328000, + 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000, + 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, + 0x38354000, 0x38358000, 0x3835C000, 0x38360000, 0x38364000, + 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000, + 0x3837C000, 0x38380000, 0x38384000, 0x38388000, 0x3838C000, + 0x38390000, 0x38394000, 0x38398000, 0x3839C000, 0x383A0000, + 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, + 0x383B8000, 0x383BC000, 0x383C0000, 0x383C4000, 0x383C8000, + 0x383CC000, 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, + 0x383E0000, 0x383E4000, 0x383E8000, 0x383EC000, 0x383F0000, + 0x383F4000, 0x383F8000, 0x383FC000, 0x38400000, 0x38404000, + 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000, + 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, + 0x38430000, 0x38434000, 0x38438000, 0x3843C000, 0x38440000, + 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000, + 0x38458000, 0x3845C000, 0x38460000, 0x38464000, 0x38468000, + 0x3846C000, 0x38470000, 0x38474000, 0x38478000, 0x3847C000, + 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000, + 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, + 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, + 0x384BC000, 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, + 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000, + 0x384E4000, 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000, + 0x384F8000, 0x384FC000, 0x38500000, 0x38504000, 0x38508000, + 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, + 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, + 0x38534000, 0x38538000, 0x3853C000, 0x38540000, 0x38544000, + 0x38548000, 0x3854C000, 0x38550000, 0x38554000, 0x38558000, + 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000, + 0x38570000, 0x38574000, 0x38578000, 0x3857C000, 0x38580000, + 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, + 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, + 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000, + 0x385C0000, 0x385C4000, 0x385C8000, 0x385CC000, 0x385D0000, + 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, 0x385E4000, + 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000, + 0x385FC000, 0x38600000, 0x38604000, 0x38608000, 0x3860C000, + 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, + 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000, + 0x38638000, 0x3863C000, 0x38640000, 0x38644000, 0x38648000, + 0x3864C000, 0x38650000, 0x38654000, 0x38658000, 0x3865C000, + 0x38660000, 0x38664000, 0x38668000, 0x3866C000, 0x38670000, + 0x38674000, 0x38678000, 0x3867C000, 0x38680000, 0x38684000, + 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, + 0x3869C000, 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, + 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000, 0x386C0000, + 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000, + 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, 0x386E8000, + 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000, + 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, + 0x38714000, 0x38718000, 0x3871C000, 0x38720000, 0x38724000, + 0x38728000, 0x3872C000, 0x38730000, 0x38734000, 0x38738000, + 0x3873C000, 0x38740000, 0x38744000, 0x38748000, 0x3874C000, + 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000, + 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, + 0x38778000, 0x3877C000, 0x38780000, 0x38784000, 0x38788000, + 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000, + 0x387A0000, 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000, + 0x387B4000, 0x387B8000, 0x387BC000, 0x387C0000, 0x387C4000, + 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000, + 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, + 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000, 0x38000000, + 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800A000, + 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000, + 0x38016000, 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000, + 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000, + 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, + 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, + 0x3803E000, 0x38040000, 0x38042000, 0x38044000, 0x38046000, + 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, 0x38050000, + 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000, + 0x3805C000, 0x3805E000, 0x38060000, 0x38062000, 0x38064000, + 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, + 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, + 0x3807A000, 0x3807C000, 0x3807E000, 0x38080000, 0x38082000, + 0x38084000, 0x38086000, 0x38088000, 0x3808A000, 0x3808C000, + 0x3808E000, 0x38090000, 0x38092000, 0x38094000, 0x38096000, + 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000, 0x380A0000, + 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, + 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, + 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000, + 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000, + 0x380CA000, 0x380CC000, 0x380CE000, 0x380D0000, 0x380D2000, + 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, 0x380DC000, + 0x380DE000, 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, + 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, + 0x380F2000, 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, + 0x380FC000, 0x380FE000, 0x38100000, 0x38102000, 0x38104000, + 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000, + 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000, + 0x3811A000, 0x3811C000, 0x3811E000, 0x38120000, 0x38122000, + 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, + 0x3812E000, 0x38130000, 0x38132000, 0x38134000, 0x38136000, + 0x38138000, 0x3813A000, 0x3813C000, 0x3813E000, 0x38140000, + 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000, + 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000, + 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000, + 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, + 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, + 0x38174000, 0x38176000, 0x38178000, 0x3817A000, 0x3817C000, + 0x3817E000, 0x38180000, 0x38182000, 0x38184000, 0x38186000, + 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000, + 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, + 0x3819C000, 0x3819E000, 0x381A0000, 0x381A2000, 0x381A4000, + 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, + 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000, + 0x381BA000, 0x381BC000, 0x381BE000, 0x381C0000, 0x381C2000, + 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, 0x381CC000, + 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, + 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000, 0x381E0000, + 0x381E2000, 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, + 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, 0x381F4000, + 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000, + 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000, + 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, + 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, + 0x3821E000, 0x38220000, 0x38222000, 0x38224000, 0x38226000, + 0x38228000, 0x3822A000, 0x3822C000, 0x3822E000, 0x38230000, + 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823A000, + 0x3823C000, 0x3823E000, 0x38240000, 0x38242000, 0x38244000, + 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, + 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, + 0x3825A000, 0x3825C000, 0x3825E000, 0x38260000, 0x38262000, + 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000, + 0x3826E000, 0x38270000, 0x38272000, 0x38274000, 0x38276000, + 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000, 0x38280000, + 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, + 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, + 0x38296000, 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000, + 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000, + 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000, + 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, 0x382BC000, + 0x382BE000, 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, + 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, + 0x382D2000, 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, + 0x382DC000, 0x382DE000, 0x382E0000, 0x382E2000, 0x382E4000, + 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000, + 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000, + 0x382FA000, 0x382FC000, 0x382FE000, 0x38300000, 0x38302000, + 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, + 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, + 0x38318000, 0x3831A000, 0x3831C000, 0x3831E000, 0x38320000, + 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832A000, + 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000, + 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000, + 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, + 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, + 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000, + 0x3835E000, 0x38360000, 0x38362000, 0x38364000, 0x38366000, + 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, 0x38370000, + 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, + 0x3837C000, 0x3837E000, 0x38380000, 0x38382000, 0x38384000, + 0x38386000, 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, + 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000, + 0x3839A000, 0x3839C000, 0x3839E000, 0x383A0000, 0x383A2000, + 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000, + 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, + 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000, 0x383C0000, + 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, + 0x383CC000, 0x383CE000, 0x383D0000, 0x383D2000, 0x383D4000, + 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, 0x383DE000, + 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000, + 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, + 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, + 0x383FE000, 0x38400000, 0x38402000, 0x38404000, 0x38406000, + 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000, + 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841A000, + 0x3841C000, 0x3841E000, 0x38420000, 0x38422000, 0x38424000, + 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, + 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, + 0x3843A000, 0x3843C000, 0x3843E000, 0x38440000, 0x38442000, + 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000, + 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, + 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000, 0x38460000, + 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, + 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, + 0x38476000, 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000, + 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000, + 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000, + 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000, + 0x3849E000, 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, + 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, + 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, + 0x384BC000, 0x384BE000, 0x384C0000, 0x384C2000, 0x384C4000, + 0x384C6000, 0x384C8000, 0x384CA000, 0x384CC000, 0x384CE000, + 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000, + 0x384DA000, 0x384DC000, 0x384DE000, 0x384E0000, 0x384E2000, + 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, + 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, + 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000, 0x38500000, + 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850A000, + 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, 0x38514000, + 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000, + 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, + 0x3852A000, 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, + 0x38534000, 0x38536000, 0x38538000, 0x3853A000, 0x3853C000, + 0x3853E000, 0x38540000, 0x38542000, 0x38544000, 0x38546000, + 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000, + 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, + 0x3855C000, 0x3855E000, 0x38560000, 0x38562000, 0x38564000, + 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, + 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000, + 0x3857A000, 0x3857C000, 0x3857E000, 0x38580000, 0x38582000, + 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000, + 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, + 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000, 0x385A0000, + 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, + 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000, + 0x385B6000, 0x385B8000, 0x385BA000, 0x385BC000, 0x385BE000, + 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, 0x385C8000, + 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, + 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, + 0x385DE000, 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, + 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000, + 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000, + 0x385FC000, 0x385FE000, 0x38600000, 0x38602000, 0x38604000, + 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, + 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, + 0x3861A000, 0x3861C000, 0x3861E000, 0x38620000, 0x38622000, + 0x38624000, 0x38626000, 0x38628000, 0x3862A000, 0x3862C000, + 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, + 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000, 0x38640000, + 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, + 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, + 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000, + 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000, + 0x3866A000, 0x3866C000, 0x3866E000, 0x38670000, 0x38672000, + 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000, + 0x3867E000, 0x38680000, 0x38682000, 0x38684000, 0x38686000, + 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, + 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869A000, + 0x3869C000, 0x3869E000, 0x386A0000, 0x386A2000, 0x386A4000, + 0x386A6000, 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000, + 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, 0x386B8000, + 0x386BA000, 0x386BC000, 0x386BE000, 0x386C0000, 0x386C2000, + 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, + 0x386CE000, 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, + 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000, 0x386E0000, + 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000, + 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000, + 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000, + 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, + 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, + 0x38714000, 0x38716000, 0x38718000, 0x3871A000, 0x3871C000, + 0x3871E000, 0x38720000, 0x38722000, 0x38724000, 0x38726000, + 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000, + 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, + 0x3873C000, 0x3873E000, 0x38740000, 0x38742000, 0x38744000, + 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, + 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, + 0x3875A000, 0x3875C000, 0x3875E000, 0x38760000, 0x38762000, + 0x38764000, 0x38766000, 0x38768000, 0x3876A000, 0x3876C000, + 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, + 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000, 0x38780000, + 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878A000, + 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000, + 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000, + 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, 0x387A8000, + 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, + 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, + 0x387BE000, 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, + 0x387C8000, 0x387CA000, 0x387CC000, 0x387CE000, 0x387D0000, + 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000, + 0x387DC000, 0x387DE000, 0x387E0000, 0x387E2000, 0x387E4000, + 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, + 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, + 0x387FA000, 0x387FC000, 0x387FE000}; + static const uint32 exponent_table[64] = { + 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, + 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, + 0x06000000, 0x06800000, 0x07000000, 0x07800000, 0x08000000, 0x08800000, + 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, + 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, + 0x0F000000, 0x47800000, 0x80000000, 0x80800000, 0x81000000, 0x81800000, + 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, + 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000, + 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, + 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, + 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000}; + static const unsigned short offset_table[64] = { + 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, + 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, + 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 0, + 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, + 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, + 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024}; + uint32 bits = mantissa_table[offset_table[value >> 10] + (value & 0x3FF)] + + exponent_table[value >> 10]; + // return *reinterpret_cast(&bits); //violating + //strict aliasing! + float out; + std::memcpy(&out, &bits, sizeof(float)); + return out; +} - /// Gamma implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr tgamma(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::tgamma(arg)); - #else - if(arg == 0.0f) - return builtin_signbit(arg) ? expr(-std::numeric_limits::infinity()) : expr(std::numeric_limits::infinity()); - if(arg < 0.0f) - { - float i, f = std::modf(-arg, &i); - if(f == 0.0f) - return expr(std::numeric_limits::quiet_NaN()); - double value = 3.1415926535897932384626433832795 / (std::sin(3.1415926535897932384626433832795*f)*std::exp(lgamma(1.0-arg))); - return expr(static_cast((std::fmod(i, 2.0f)==0.0f) ? -value : value)); - } - if(builtin_isinf(arg)) - return expr(arg); - return expr(static_cast(std::exp(lgamma(static_cast(arg))))); - #endif - } +/// Convert half-precision to IEEE double-precision. +/// \param value binary representation of half-precision value +/// \return double-precision value +inline double half2float_impl(uint16 value, double, true_type) { + typedef bits::type uint32; + typedef bits::type uint64; + uint32 hi = static_cast(value & 0x8000) << 16; + int abs = value & 0x7FFF; + if (abs) { + hi |= 0x3F000000 << static_cast(abs >= 0x7C00); + for (; abs < 0x400; abs <<= 1, hi -= 0x100000) + ; + hi += static_cast(abs) << 10; + } + uint64 bits = static_cast(hi) << 32; + // return *reinterpret_cast(&bits); //violating + //strict aliasing! + double out; + std::memcpy(&out, &bits, sizeof(double)); + return out; +} - /// Floor implementation. - /// \param arg value to round - /// \return rounded value - static half floor(half arg) { return half(binary, round_half(arg.data_)); } - - /// Ceiling implementation. - /// \param arg value to round - /// \return rounded value - static half ceil(half arg) { return half(binary, round_half(arg.data_)); } - - /// Truncation implementation. - /// \param arg value to round - /// \return rounded value - static half trunc(half arg) { return half(binary, round_half(arg.data_)); } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static half round(half arg) { return half(binary, round_half_up(arg.data_)); } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static long lround(half arg) { return detail::half2int_up(arg.data_); } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static half rint(half arg) { return half(binary, round_half(arg.data_)); } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static long lrint(half arg) { return detail::half2int(arg.data_); } - - #if HALF_ENABLE_CPP11_LONG_LONG - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static long long llround(half arg) { return detail::half2int_up(arg.data_); } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static long long llrint(half arg) { return detail::half2int(arg.data_); } - #endif - - /// Decompression implementation. - /// \param arg number to decompress - /// \param exp address to store exponent at - /// \return normalized significant - static half frexp(half arg, int *exp) - { - int m = arg.data_ & 0x7FFF, e = -14; - if(m >= 0x7C00 || !m) - return *exp = 0, arg; - for(; m<0x400; m<<=1,--e) ; - return *exp = e+(m>>10), half(binary, (arg.data_&0x8000)|0x3800|(m&0x3FF)); - } +/// Convert half-precision to non-IEEE floating point. +/// \tparam T type to convert to (builtin integer type) +/// \param value binary representation of half-precision value +/// \return floating point value +template +T half2float_impl(uint16 value, T, ...) { + T out; + int abs = value & 0x7FFF; + if (abs > 0x7C00) + out = std::numeric_limits::has_quiet_NaN + ? std::numeric_limits::quiet_NaN() + : T(); + else if (abs == 0x7C00) + out = std::numeric_limits::has_infinity + ? std::numeric_limits::infinity() + : std::numeric_limits::max(); + else if (abs > 0x3FF) + out = std::ldexp(static_cast((abs & 0x3FF) | 0x400), (abs >> 10) - 25); + else + out = std::ldexp(static_cast(abs), -24); + return (value & 0x8000) ? -out : out; +} - /// Decompression implementation. - /// \param arg number to decompress - /// \param iptr address to store integer part at - /// \return fractional part - static half modf(half arg, half *iptr) - { - unsigned int e = arg.data_ & 0x7FFF; - if(e >= 0x6400) - return *iptr = arg, half(binary, arg.data_&(0x8000U|-(e>0x7C00))); - if(e < 0x3C00) - return iptr->data_ = arg.data_ & 0x8000, arg; - e >>= 10; - unsigned int mask = (1<<(25-e)) - 1, m = arg.data_ & mask; - iptr->data_ = arg.data_ & ~mask; - if(!m) - return half(binary, arg.data_&0x8000); - for(; m<0x400; m<<=1,--e) ; - return half(binary, static_cast((arg.data_&0x8000)|(e<<10)|(m&0x3FF))); - } +/// Convert half-precision to floating point. +/// \tparam T type to convert to (builtin integer type) +/// \param value binary representation of half-precision value +/// \return floating point value +template +T half2float(uint16 value) { + return half2float_impl(value, T(), + bool_type < std::numeric_limits::is_iec559 && + sizeof(typename bits::type) == sizeof(T) > ()); +} - /// Scaling implementation. - /// \param arg number to scale - /// \param exp power of two to scale by - /// \return scaled number - static half scalbln(half arg, long exp) - { - unsigned int m = arg.data_ & 0x7FFF; - if(m >= 0x7C00 || !m) - return arg; - for(; m<0x400; m<<=1,--exp) ; - exp += m >> 10; - uint16 value = arg.data_ & 0x8000; - if(exp > 30) - { - if(half::round_style == std::round_toward_zero) - value |= 0x7BFF; - else if(half::round_style == std::round_toward_infinity) - value |= 0x7C00 - (value>>15); - else if(half::round_style == std::round_toward_neg_infinity) - value |= 0x7BFF + (value>>15); - else - value |= 0x7C00; - } - else if(exp > 0) - value |= (exp<<10) | (m&0x3FF); - else if(exp > -11) - { - m = (m&0x3FF) | 0x400; - if(half::round_style == std::round_to_nearest) - { - m += 1 << -exp; - #if HALF_ROUND_TIES_TO_EVEN - m -= (m>>(1-exp)) & 1; - #endif - } - else if(half::round_style == std::round_toward_infinity) - m += ((value>>15)-1) & ((1<<(1-exp))-1U); - else if(half::round_style == std::round_toward_neg_infinity) - m += -(value>>15) & ((1<<(1-exp))-1U); - value |= m >> (1-exp); - } - else if(half::round_style == std::round_toward_infinity) - value -= (value>>15) - 1; - else if(half::round_style == std::round_toward_neg_infinity) - value += value >> 15; - return half(binary, value); - } +/// Convert half-precision floating point to integer. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding +/// \tparam E `true` for round to even, `false` for round away from zero +/// \tparam T type to convert to (buitlin integer type with at least 16 bits +/// precision, excluding any implicit sign bits) +/// \param value binary representation of half-precision value +/// \return integral value +template +T half2int_impl(uint16 value) { +#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_integral::value, + "half to int conversion only supports builtin integer types"); +#endif + unsigned int e = value & 0x7FFF; + if (e >= 0x7C00) + return (value & 0x8000) ? std::numeric_limits::min() + : std::numeric_limits::max(); + if (e < 0x3800) { + if (R == std::round_toward_infinity) + return T(~(value >> 15) & (e != 0)); + else if (R == std::round_toward_neg_infinity) + return -T(value > 0x8000); + return T(); + } + unsigned int m = (value & 0x3FF) | 0x400; + e >>= 10; + if (e < 25) { + if (R == std::round_to_nearest) + m += (1 << (24 - e)) - (~(m >> (25 - e)) & E); + else if (R == std::round_toward_infinity) + m += ((value >> 15) - 1) & ((1 << (25 - e)) - 1U); + else if (R == std::round_toward_neg_infinity) + m += -(value >> 15) & ((1 << (25 - e)) - 1U); + m >>= 25 - e; + } else + m <<= e - 25; + return (value & 0x8000) ? -static_cast(m) : static_cast(m); +} - /// Exponent implementation. - /// \param arg number to query - /// \return floating point exponent - static int ilogb(half arg) - { - int abs = arg.data_ & 0x7FFF; - if(!abs) - return FP_ILOGB0; - if(abs < 0x7C00) - { - int exp = (abs>>10) - 15; - if(abs < 0x400) - for(; abs<0x200; abs<<=1,--exp) ; - return exp; - } - if(abs > 0x7C00) - return FP_ILOGBNAN; - return INT_MAX; - } +/// Convert half-precision floating point to integer. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding +/// \tparam T type to convert to (buitlin integer type with at least 16 bits +/// precision, excluding any implicit sign bits) +/// \param value binary representation of half-precision value +/// \return integral value +template +T half2int(uint16 value) { + return half2int_impl(value); +} - /// Exponent implementation. - /// \param arg number to query - /// \return floating point exponent - static half logb(half arg) - { - int abs = arg.data_ & 0x7FFF; - if(!abs) - return half(binary, 0xFC00); - if(abs < 0x7C00) - { - int exp = (abs>>10) - 15; - if(abs < 0x400) - for(; abs<0x200; abs<<=1,--exp) ; - uint16 bits = (exp<0) << 15; - if(exp) - { - unsigned int m = std::abs(exp) << 6, e = 18; - for(; m<0x400; m<<=1,--e) ; - bits |= (e<<10) + m; - } - return half(binary, bits); - } - if(abs > 0x7C00) - return arg; - return half(binary, 0x7C00); - } +/// Convert half-precision floating point to integer using +/// round-to-nearest-away-from-zero. +/// \tparam T type to convert to (buitlin integer type with at least 16 bits +/// precision, excluding any implicit sign bits) +/// \param value binary representation of half-precision value +/// \return integral value +template +T half2int_up(uint16 value) { + return half2int_impl(value); +} - /// Enumeration implementation. - /// \param from number to increase/decrease - /// \param to direction to enumerate into - /// \return next representable number - static half nextafter(half from, half to) - { - uint16 fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF; - if(fabs > 0x7C00) - return from; - if(tabs > 0x7C00 || from.data_ == to.data_ || !(fabs|tabs)) - return to; - if(!fabs) - return half(binary, (to.data_&0x8000)+1); - bool lt = ((fabs==from.data_) ? static_cast(fabs) : -static_cast(fabs)) < - ((tabs==to.data_) ? static_cast(tabs) : -static_cast(tabs)); - return half(binary, from.data_+(((from.data_>>15)^static_cast(lt))<<1)-1); - } +/// Round half-precision number to nearest integer value. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding +/// \tparam E `true` for round to even, `false` for round away from zero +/// \param value binary representation of half-precision value +/// \return half-precision bits for nearest integral value +template +uint16 round_half_impl(uint16 value) { + unsigned int e = value & 0x7FFF; + uint16 result = value; + if (e < 0x3C00) { + result &= 0x8000; + if (R == std::round_to_nearest) + result |= 0x3C00U & -(e >= (0x3800 + E)); + else if (R == std::round_toward_infinity) + result |= 0x3C00U & -(~(value >> 15) & (e != 0)); + else if (R == std::round_toward_neg_infinity) + result |= 0x3C00U & -(value > 0x8000); + } else if (e < 0x6400) { + e = 25 - (e >> 10); + unsigned int mask = (1 << e) - 1; + if (R == std::round_to_nearest) + result += (1 << (e - 1)) - (~(result >> e) & E); + else if (R == std::round_toward_infinity) + result += mask & ((value >> 15) - 1); + else if (R == std::round_toward_neg_infinity) + result += mask & -(value >> 15); + result &= ~mask; + } + return result; +} - /// Enumeration implementation. - /// \param from number to increase/decrease - /// \param to direction to enumerate into - /// \return next representable number - static half nexttoward(half from, long double to) - { - if(isnan(from)) - return from; - long double lfrom = static_cast(from); - if(builtin_isnan(to) || lfrom == to) - return half(static_cast(to)); - if(!(from.data_&0x7FFF)) - return half(binary, (static_cast(builtin_signbit(to))<<15)+1); - return half(binary, from.data_+(((from.data_>>15)^static_cast(lfrom +uint16 round_half(uint16 value) { + return round_half_impl(value); +} - /// Sign implementation - /// \param x first operand - /// \param y second operand - /// \return composed value - static half copysign(half x, half y) { return half(binary, x.data_^((x.data_^y.data_)&0x8000)); } - - /// Classification implementation. - /// \param arg value to classify - /// \retval true if infinite number - /// \retval false else - static int fpclassify(half arg) - { - unsigned int abs = arg.data_ & 0x7FFF; - return abs ? ((abs>0x3FF) ? ((abs>=0x7C00) ? ((abs>0x7C00) ? FP_NAN : FP_INFINITE) : FP_NORMAL) :FP_SUBNORMAL) : FP_ZERO; - } +/// Round half-precision number to nearest integer value using +/// round-to-nearest-away-from-zero. +/// \param value binary representation of half-precision value +/// \return half-precision bits for nearest integral value +inline uint16 round_half_up(uint16 value) { + return round_half_impl(value); +} +/// \} + +struct functions; +template +struct unary_specialized; +template +struct binary_specialized; +template +struct half_caster; +} - /// Classification implementation. - /// \param arg value to classify - /// \retval true if finite number - /// \retval false else - static bool isfinite(half arg) { return (arg.data_&0x7C00) != 0x7C00; } - - /// Classification implementation. - /// \param arg value to classify - /// \retval true if infinite number - /// \retval false else - static bool isinf(half arg) { return (arg.data_&0x7FFF) == 0x7C00; } - - /// Classification implementation. - /// \param arg value to classify - /// \retval true if not a number - /// \retval false else - static bool isnan(half arg) { return (arg.data_&0x7FFF) > 0x7C00; } - - /// Classification implementation. - /// \param arg value to classify - /// \retval true if normal number - /// \retval false else - static bool isnormal(half arg) { return ((arg.data_&0x7C00)!=0) & ((arg.data_&0x7C00)!=0x7C00); } - - /// Sign bit implementation. - /// \param arg value to check - /// \retval true if signed - /// \retval false if unsigned - static bool signbit(half arg) { return (arg.data_&0x8000) != 0; } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if operands equal - /// \retval false else - static bool isequal(half x, half y) { return (x.data_==y.data_ || !((x.data_|y.data_)&0x7FFF)) && !isnan(x); } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if operands not equal - /// \retval false else - static bool isnotequal(half x, half y) { return (x.data_!=y.data_ && ((x.data_|y.data_)&0x7FFF)) || isnan(x); } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x > \a y - /// \retval false else - static bool isgreater(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs)); - } +/// Half-precision floating point type. +/// This class implements an IEEE-conformant half-precision floating point type +/// with the usual arithmetic operators and +/// conversions. It is implicitly convertible to single-precision floating +/// point, which makes artihmetic expressions and +/// functions with mixed-type operands to be of the most precise operand type. +/// Additionally all arithmetic operations +/// (and many mathematical functions) are carried out in single-precision +/// internally. All conversions from single- to +/// half-precision are done using the library's default rounding mode, but +/// temporary results inside chained arithmetic +/// expressions are kept in single-precision as long as possible (while of +/// course still maintaining a strong half-precision type). +/// +/// According to the C++98/03 definition, the half type is not a POD type. But +/// according to C++11's less strict and +/// extended definitions it is both a standard layout type and a trivially +/// copyable type (even if not a POD type), which +/// means it can be standard-conformantly copied using raw binary copies. But in +/// this context some more words about the +/// actual size of the type. Although the half is representing an IEEE 16-bit +/// type, it does not neccessarily have to be of +/// exactly 16-bits size. But on any reasonable implementation the actual binary +/// representation of this type will most +/// probably not ivolve any additional "magic" or padding beyond the simple +/// binary representation of the underlying 16-bit +/// IEEE number, even if not strictly guaranteed by the standard. But even then +/// it only has an actual size of 16 bits if +/// your C++ implementation supports an unsigned integer type of exactly 16 bits +/// width. But this should be the case on +/// nearly any reasonable platform. +/// +/// So if your C++ implementation is not totally exotic or imposes special +/// alignment requirements, it is a reasonable +/// assumption that the data of a half is just comprised of the 2 bytes of the +/// underlying IEEE representation. +class half { + friend struct detail::functions; + friend struct detail::unary_specialized; + friend struct detail::binary_specialized; + template + friend struct detail::half_caster; + friend class std::numeric_limits; +#if HALF_ENABLE_CPP11_HASH + friend struct std::hash; +#endif +#if HALF_ENABLE_CPP11_USER_LITERALS + friend half literal::operator"" _h(long double); +#endif - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x >= \a y - /// \retval false else - static bool isgreaterequal(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) >= ((yabs==y.data_) ? yabs : -yabs)); - } + public: + /// Default constructor. + /// This initializes the half to 0. Although this does not match the builtin + /// types' default-initialization semantics + /// and may be less efficient than no initialization, it is needed to provide + /// proper value-initialization semantics. + HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {} + + /// Copy constructor. + /// \tparam T type of concrete half expression + /// \param rhs half expression to copy from + half(detail::expr rhs) + : data_(detail::float2half(static_cast(rhs))) {} + + /// Conversion constructor. + /// \param rhs float to convert + explicit half(float rhs) : data_(detail::float2half(rhs)) {} + + /// Conversion to single-precision. + /// \return single precision value representing expression value + operator float() const { return detail::half2float(data_); } + + /// Assignment operator. + /// \tparam T type of concrete half expression + /// \param rhs half expression to copy from + /// \return reference to this half + half &operator=(detail::expr rhs) { return *this = static_cast(rhs); } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to add + /// \return reference to this half + template + typename detail::enable::type operator+=(T rhs) { + return *this += static_cast(rhs); + } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to subtract + /// \return reference to this half + template + typename detail::enable::type operator-=(T rhs) { + return *this -= static_cast(rhs); + } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to multiply with + /// \return reference to this half + template + typename detail::enable::type operator*=(T rhs) { + return *this *= static_cast(rhs); + } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to divide by + /// \return reference to this half + template + typename detail::enable::type operator/=(T rhs) { + return *this /= static_cast(rhs); + } + + /// Assignment operator. + /// \param rhs single-precision value to copy from + /// \return reference to this half + half &operator=(float rhs) { + data_ = detail::float2half(rhs); + return *this; + } + + /// Arithmetic assignment. + /// \param rhs single-precision value to add + /// \return reference to this half + half &operator+=(float rhs) { + data_ = + detail::float2half(detail::half2float(data_) + rhs); + return *this; + } + + /// Arithmetic assignment. + /// \param rhs single-precision value to subtract + /// \return reference to this half + half &operator-=(float rhs) { + data_ = + detail::float2half(detail::half2float(data_) - rhs); + return *this; + } + + /// Arithmetic assignment. + /// \param rhs single-precision value to multiply with + /// \return reference to this half + half &operator*=(float rhs) { + data_ = + detail::float2half(detail::half2float(data_) * rhs); + return *this; + } + + /// Arithmetic assignment. + /// \param rhs single-precision value to divide by + /// \return reference to this half + half &operator/=(float rhs) { + data_ = + detail::float2half(detail::half2float(data_) / rhs); + return *this; + } + + /// Prefix increment. + /// \return incremented half value + half &operator++() { return *this += 1.0f; } + + /// Prefix decrement. + /// \return decremented half value + half &operator--() { return *this -= 1.0f; } + + /// Postfix increment. + /// \return non-incremented half value + half operator++(int) { + half out(*this); + ++*this; + return out; + } + + /// Postfix decrement. + /// \return non-decremented half value + half operator--(int) { + half out(*this); + --*this; + return out; + } + + private: + /// Rounding mode to use + static const std::float_round_style round_style = + (std::float_round_style)(HALF_ROUND_STYLE); + + /// Constructor. + /// \param bits binary representation to set half to + HALF_CONSTEXPR half(detail::binary_t, detail::uint16 bits) HALF_NOEXCEPT + : data_(bits) {} + + /// Internal binary representation + detail::uint16 data_; +}; - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x < \a y - /// \retval false else - static bool isless(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs)); - } +#if HALF_ENABLE_CPP11_USER_LITERALS +namespace literal { +/// Half literal. +/// While this returns an actual half-precision value, half literals can +/// unfortunately not be constant expressions due +/// to rather involved conversions. +/// \param value literal value +/// \return half with given value (if representable) +inline half operator"" _h(long double value) { + return half(detail::binary, detail::float2half(value)); +} +} +#endif - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x <= \a y - /// \retval false else - static bool islessequal(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) <= ((yabs==y.data_) ? yabs : -yabs)); - } +namespace detail { +/// Wrapper implementing unspecialized half-precision functions. +struct functions { + /// Addition implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision sum stored in single-precision + static expr plus(float x, float y) { return expr(x + y); } + + /// Subtraction implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision difference stored in single-precision + static expr minus(float x, float y) { return expr(x - y); } + + /// Multiplication implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision product stored in single-precision + static expr multiplies(float x, float y) { return expr(x * y); } + + /// Division implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision quotient stored in single-precision + static expr divides(float x, float y) { return expr(x / y); } + + /// Output implementation. + /// \param out stream to write to + /// \param arg value to write + /// \return reference to stream + template + static std::basic_ostream &write( + std::basic_ostream &out, float arg) { + return out << arg; + } + + /// Input implementation. + /// \param in stream to read from + /// \param arg half to read into + /// \return reference to stream + template + static std::basic_istream &read( + std::basic_istream &in, half &arg) { + float f; + if (in >> f) arg = f; + return in; + } + + /// Modulo implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision division remainder stored in single-precision + static expr fmod(float x, float y) { return expr(std::fmod(x, y)); } + + /// Remainder implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision division remainder stored in single-precision + static expr remainder(float x, float y) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::remainder(x, y)); +#else + if (builtin_isnan(x) || builtin_isnan(y)) + return expr(std::numeric_limits::quiet_NaN()); + float ax = std::fabs(x), ay = std::fabs(y); + if (ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) + return expr(std::numeric_limits::quiet_NaN()); + if (ay >= 65536.0f) return expr(x); + if (ax == ay) return expr(builtin_signbit(x) ? -0.0f : 0.0f); + ax = std::fmod(ax, ay + ay); + float y2 = 0.5f * ay; + if (ax > y2) { + ax -= ay; + if (ax >= y2) ax -= ay; + } + return expr(builtin_signbit(x) ? -ax : ax); +#endif + } + + /// Remainder implementation. + /// \param x first operand + /// \param y second operand + /// \param quo address to store quotient bits at + /// \return Half-precision division remainder stored in single-precision + static expr remquo(float x, float y, int *quo) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::remquo(x, y, quo)); +#else + if (builtin_isnan(x) || builtin_isnan(y)) + return expr(std::numeric_limits::quiet_NaN()); + bool sign = builtin_signbit(x), + qsign = static_cast(sign ^ builtin_signbit(y)); + float ax = std::fabs(x), ay = std::fabs(y); + if (ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) + return expr(std::numeric_limits::quiet_NaN()); + if (ay >= 65536.0f) return expr(x); + if (ax == ay) return *quo = qsign ? -1 : 1, expr(sign ? -0.0f : 0.0f); + ax = std::fmod(ax, 8.0f * ay); + int cquo = 0; + if (ax >= 4.0f * ay) { + ax -= 4.0f * ay; + cquo += 4; + } + if (ax >= 2.0f * ay) { + ax -= 2.0f * ay; + cquo += 2; + } + float y2 = 0.5f * ay; + if (ax > y2) { + ax -= ay; + ++cquo; + if (ax >= y2) { + ax -= ay; + ++cquo; + } + } + return *quo = qsign ? -cquo : cquo, expr(sign ? -ax : ax); +#endif + } + + /// Positive difference implementation. + /// \param x first operand + /// \param y second operand + /// \return Positive difference stored in single-precision + static expr fdim(float x, float y) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::fdim(x, y)); +#else + return expr((x <= y) ? 0.0f : (x - y)); +#endif + } + + /// Fused multiply-add implementation. + /// \param x first operand + /// \param y second operand + /// \param z third operand + /// \return \a x * \a y + \a z stored in single-precision + static expr fma(float x, float y, float z) { +#if HALF_ENABLE_CPP11_CMATH && defined(FP_FAST_FMAF) + return expr(std::fma(x, y, z)); +#else + return expr(x * y + z); +#endif + } + + /// Get NaN. + /// \return Half-precision quiet NaN + static half nanh() { return half(binary, 0x7FFF); } + + /// Exponential implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr exp(float arg) { return expr(std::exp(arg)); } + + /// Exponential implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr expm1(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::expm1(arg)); +#else + return expr(static_cast(std::exp(static_cast(arg)) - 1.0)); +#endif + } + + /// Binary exponential implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr exp2(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::exp2(arg)); +#else + return expr( + static_cast(std::exp(arg * 0.69314718055994530941723212145818))); +#endif + } + + /// Logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log(float arg) { return expr(std::log(arg)); } + + /// Common logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log10(float arg) { return expr(std::log10(arg)); } + + /// Logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log1p(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::log1p(arg)); +#else + return expr(static_cast(std::log(1.0 + arg))); +#endif + } + + /// Binary logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log2(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::log2(arg)); +#else + return expr(static_cast(std::log(static_cast(arg)) * + 1.4426950408889634073599246810019)); +#endif + } + + /// Square root implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr sqrt(float arg) { return expr(std::sqrt(arg)); } + + /// Cubic root implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr cbrt(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::cbrt(arg)); +#else + if (builtin_isnan(arg) || builtin_isinf(arg)) return expr(arg); + return expr(builtin_signbit(arg) + ? -static_cast( + std::pow(-static_cast(arg), 1.0 / 3.0)) + : static_cast( + std::pow(static_cast(arg), 1.0 / 3.0))); +#endif + } + + /// Hypotenuse implementation. + /// \param x first argument + /// \param y second argument + /// \return function value stored in single-preicision + static expr hypot(float x, float y) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::hypot(x, y)); +#else + return expr( + (builtin_isinf(x) || builtin_isinf(y)) + ? std::numeric_limits::infinity() + : static_cast(std::sqrt(static_cast(x) * x + + static_cast(y) * y))); +#endif + } + + /// Power implementation. + /// \param base value to exponentiate + /// \param exp power to expontiate to + /// \return function value stored in single-preicision + static expr pow(float base, float exp) { return expr(std::pow(base, exp)); } + + /// Sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr sin(float arg) { return expr(std::sin(arg)); } + + /// Cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr cos(float arg) { return expr(std::cos(arg)); } + + /// Tan implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr tan(float arg) { return expr(std::tan(arg)); } + + /// Arc sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr asin(float arg) { return expr(std::asin(arg)); } + + /// Arc cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr acos(float arg) { return expr(std::acos(arg)); } + + /// Arc tangent implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr atan(float arg) { return expr(std::atan(arg)); } + + /// Arc tangent implementation. + /// \param x first argument + /// \param y second argument + /// \return function value stored in single-preicision + static expr atan2(float x, float y) { return expr(std::atan2(x, y)); } + + /// Hyperbolic sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr sinh(float arg) { return expr(std::sinh(arg)); } + + /// Hyperbolic cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr cosh(float arg) { return expr(std::cosh(arg)); } + + /// Hyperbolic tangent implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr tanh(float arg) { return expr(std::tanh(arg)); } + + /// Hyperbolic area sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr asinh(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::asinh(arg)); +#else + return expr( + (arg == -std::numeric_limits::infinity()) + ? arg + : static_cast(std::log(arg + std::sqrt(arg * arg + 1.0)))); +#endif + } + + /// Hyperbolic area cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr acosh(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::acosh(arg)); +#else + return expr((arg < -1.0f) ? std::numeric_limits::quiet_NaN() + : static_cast(std::log( + arg + std::sqrt(arg * arg - 1.0)))); +#endif + } + + /// Hyperbolic area tangent implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr atanh(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::atanh(arg)); +#else + return expr(static_cast(0.5 * std::log((1.0 + arg) / (1.0 - arg)))); +#endif + } + + /// Error function implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr erf(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::erf(arg)); +#else + return expr(static_cast(erf(static_cast(arg)))); +#endif + } + + /// Complementary implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr erfc(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::erfc(arg)); +#else + return expr(static_cast(1.0 - erf(static_cast(arg)))); +#endif + } + + /// Gamma logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr lgamma(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::lgamma(arg)); +#else + if (builtin_isinf(arg)) return expr(std::numeric_limits::infinity()); + if (arg < 0.0f) { + float i, f = std::modf(-arg, &i); + if (f == 0.0f) return expr(std::numeric_limits::infinity()); + return expr(static_cast( + 1.1447298858494001741434273513531 - + std::log(std::abs(std::sin(3.1415926535897932384626433832795 * f))) - + lgamma(1.0 - arg))); + } + return expr(static_cast(lgamma(static_cast(arg)))); +#endif + } + + /// Gamma implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr tgamma(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::tgamma(arg)); +#else + if (arg == 0.0f) + return builtin_signbit(arg) + ? expr(-std::numeric_limits::infinity()) + : expr(std::numeric_limits::infinity()); + if (arg < 0.0f) { + float i, f = std::modf(-arg, &i); + if (f == 0.0f) return expr(std::numeric_limits::quiet_NaN()); + double value = 3.1415926535897932384626433832795 / + (std::sin(3.1415926535897932384626433832795 * f) * + std::exp(lgamma(1.0 - arg))); + return expr( + static_cast((std::fmod(i, 2.0f) == 0.0f) ? -value : value)); + } + if (builtin_isinf(arg)) return expr(arg); + return expr(static_cast(std::exp(lgamma(static_cast(arg))))); +#endif + } + + /// Floor implementation. + /// \param arg value to round + /// \return rounded value + static half floor(half arg) { + return half(binary, round_half(arg.data_)); + } + + /// Ceiling implementation. + /// \param arg value to round + /// \return rounded value + static half ceil(half arg) { + return half(binary, round_half(arg.data_)); + } + + /// Truncation implementation. + /// \param arg value to round + /// \return rounded value + static half trunc(half arg) { + return half(binary, round_half(arg.data_)); + } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static half round(half arg) { return half(binary, round_half_up(arg.data_)); } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long lround(half arg) { return detail::half2int_up(arg.data_); } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static half rint(half arg) { + return half(binary, round_half(arg.data_)); + } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long lrint(half arg) { + return detail::half2int(arg.data_); + } - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if either \a x > \a y nor \a x < \a y - /// \retval false else - static bool islessgreater(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - if(xabs > 0x7C00 || yabs > 0x7C00) - return false; - int a = (xabs==x.data_) ? xabs : -xabs, b = (yabs==y.data_) ? yabs : -yabs; - return a < b || a > b; - } +#if HALF_ENABLE_CPP11_LONG_LONG + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long long llround(half arg) { + return detail::half2int_up(arg.data_); + } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long long llrint(half arg) { + return detail::half2int(arg.data_); + } +#endif - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if operand unordered - /// \retval false else - static bool isunordered(half x, half y) { return isnan(x) || isnan(y); } + /// Decompression implementation. + /// \param arg number to decompress + /// \param exp address to store exponent at + /// \return normalized significant + static half frexp(half arg, int *exp) { + int m = arg.data_ & 0x7FFF, e = -14; + if (m >= 0x7C00 || !m) return *exp = 0, arg; + for (; m < 0x400; m <<= 1, --e) + ; + return *exp = e + (m >> 10), + half(binary, (arg.data_ & 0x8000) | 0x3800 | (m & 0x3FF)); + } + + /// Decompression implementation. + /// \param arg number to decompress + /// \param iptr address to store integer part at + /// \return fractional part + static half modf(half arg, half *iptr) { + unsigned int e = arg.data_ & 0x7FFF; + if (e >= 0x6400) + return *iptr = arg, half(binary, arg.data_ & (0x8000U | -(e > 0x7C00))); + if (e < 0x3C00) return iptr->data_ = arg.data_ & 0x8000, arg; + e >>= 10; + unsigned int mask = (1 << (25 - e)) - 1, m = arg.data_ & mask; + iptr->data_ = arg.data_ & ~mask; + if (!m) return half(binary, arg.data_ & 0x8000); + for (; m < 0x400; m <<= 1, --e) + ; + return half(binary, static_cast((arg.data_ & 0x8000) | (e << 10) | + (m & 0x3FF))); + } + + /// Scaling implementation. + /// \param arg number to scale + /// \param exp power of two to scale by + /// \return scaled number + static half scalbln(half arg, long exp) { + unsigned int m = arg.data_ & 0x7FFF; + if (m >= 0x7C00 || !m) return arg; + for (; m < 0x400; m <<= 1, --exp) + ; + exp += m >> 10; + uint16 value = arg.data_ & 0x8000; + if (exp > 30) { + if (half::round_style == std::round_toward_zero) + value |= 0x7BFF; + else if (half::round_style == std::round_toward_infinity) + value |= 0x7C00 - (value >> 15); + else if (half::round_style == std::round_toward_neg_infinity) + value |= 0x7BFF + (value >> 15); + else + value |= 0x7C00; + } else if (exp > 0) + value |= (exp << 10) | (m & 0x3FF); + else if (exp > -11) { + m = (m & 0x3FF) | 0x400; + if (half::round_style == std::round_to_nearest) { + m += 1 << -exp; +#if HALF_ROUND_TIES_TO_EVEN + m -= (m >> (1 - exp)) & 1; +#endif + } else if (half::round_style == std::round_toward_infinity) + m += ((value >> 15) - 1) & ((1 << (1 - exp)) - 1U); + else if (half::round_style == std::round_toward_neg_infinity) + m += -(value >> 15) & ((1 << (1 - exp)) - 1U); + value |= m >> (1 - exp); + } else if (half::round_style == std::round_toward_infinity) + value -= (value >> 15) - 1; + else if (half::round_style == std::round_toward_neg_infinity) + value += value >> 15; + return half(binary, value); + } + + /// Exponent implementation. + /// \param arg number to query + /// \return floating point exponent + static int ilogb(half arg) { + int abs = arg.data_ & 0x7FFF; + if (!abs) return FP_ILOGB0; + if (abs < 0x7C00) { + int exp = (abs >> 10) - 15; + if (abs < 0x400) + for (; abs < 0x200; abs <<= 1, --exp) + ; + return exp; + } + if (abs > 0x7C00) return FP_ILOGBNAN; + return INT_MAX; + } + + /// Exponent implementation. + /// \param arg number to query + /// \return floating point exponent + static half logb(half arg) { + int abs = arg.data_ & 0x7FFF; + if (!abs) return half(binary, 0xFC00); + if (abs < 0x7C00) { + int exp = (abs >> 10) - 15; + if (abs < 0x400) + for (; abs < 0x200; abs <<= 1, --exp) + ; + uint16 bits = (exp < 0) << 15; + if (exp) { + unsigned int m = std::abs(exp) << 6, e = 18; + for (; m < 0x400; m <<= 1, --e) + ; + bits |= (e << 10) + m; + } + return half(binary, bits); + } + if (abs > 0x7C00) return arg; + return half(binary, 0x7C00); + } + + /// Enumeration implementation. + /// \param from number to increase/decrease + /// \param to direction to enumerate into + /// \return next representable number + static half nextafter(half from, half to) { + uint16 fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF; + if (fabs > 0x7C00) return from; + if (tabs > 0x7C00 || from.data_ == to.data_ || !(fabs | tabs)) return to; + if (!fabs) return half(binary, (to.data_ & 0x8000) + 1); + bool lt = + ((fabs == from.data_) ? static_cast(fabs) + : -static_cast(fabs)) < + ((tabs == to.data_) ? static_cast(tabs) : -static_cast(tabs)); + return half(binary, + from.data_ + + (((from.data_ >> 15) ^ static_cast(lt)) << 1) - + 1); + } + + /// Enumeration implementation. + /// \param from number to increase/decrease + /// \param to direction to enumerate into + /// \return next representable number + static half nexttoward(half from, long double to) { + if (isnan(from)) return from; + long double lfrom = static_cast(from); + if (builtin_isnan(to) || lfrom == to) return half(static_cast(to)); + if (!(from.data_ & 0x7FFF)) + return half(binary, + (static_cast(builtin_signbit(to)) << 15) + 1); + return half( + binary, + from.data_ + + (((from.data_ >> 15) ^ static_cast(lfrom < to)) << 1) - + 1); + } + + /// Sign implementation + /// \param x first operand + /// \param y second operand + /// \return composed value + static half copysign(half x, half y) { + return half(binary, x.data_ ^ ((x.data_ ^ y.data_) & 0x8000)); + } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if infinite number + /// \retval false else + static int fpclassify(half arg) { + unsigned int abs = arg.data_ & 0x7FFF; + return abs ? ((abs > 0x3FF) ? ((abs >= 0x7C00) + ? ((abs > 0x7C00) ? FP_NAN : FP_INFINITE) + : FP_NORMAL) + : FP_SUBNORMAL) + : FP_ZERO; + } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if finite number + /// \retval false else + static bool isfinite(half arg) { return (arg.data_ & 0x7C00) != 0x7C00; } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if infinite number + /// \retval false else + static bool isinf(half arg) { return (arg.data_ & 0x7FFF) == 0x7C00; } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if not a number + /// \retval false else + static bool isnan(half arg) { return (arg.data_ & 0x7FFF) > 0x7C00; } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if normal number + /// \retval false else + static bool isnormal(half arg) { + return ((arg.data_ & 0x7C00) != 0) & ((arg.data_ & 0x7C00) != 0x7C00); + } + + /// Sign bit implementation. + /// \param arg value to check + /// \retval true if signed + /// \retval false if unsigned + static bool signbit(half arg) { return (arg.data_ & 0x8000) != 0; } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if operands equal + /// \retval false else + static bool isequal(half x, half y) { + return (x.data_ == y.data_ || !((x.data_ | y.data_) & 0x7FFF)) && !isnan(x); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if operands not equal + /// \retval false else + static bool isnotequal(half x, half y) { + return (x.data_ != y.data_ && ((x.data_ | y.data_) & 0x7FFF)) || isnan(x); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x > \a y + /// \retval false else + static bool isgreater(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs <= 0x7C00 && yabs <= 0x7C00 && + (((xabs == x.data_) ? xabs : -xabs) > + ((yabs == y.data_) ? yabs : -yabs)); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x >= \a y + /// \retval false else + static bool isgreaterequal(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs <= 0x7C00 && yabs <= 0x7C00 && + (((xabs == x.data_) ? xabs : -xabs) >= + ((yabs == y.data_) ? yabs : -yabs)); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x < \a y + /// \retval false else + static bool isless(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs <= 0x7C00 && yabs <= 0x7C00 && + (((xabs == x.data_) ? xabs : -xabs) < + ((yabs == y.data_) ? yabs : -yabs)); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x <= \a y + /// \retval false else + static bool islessequal(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs <= 0x7C00 && yabs <= 0x7C00 && + (((xabs == x.data_) ? xabs : -xabs) <= + ((yabs == y.data_) ? yabs : -yabs)); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if either \a x > \a y nor \a x < \a y + /// \retval false else + static bool islessgreater(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + if (xabs > 0x7C00 || yabs > 0x7C00) return false; + int a = (xabs == x.data_) ? xabs : -xabs, + b = (yabs == y.data_) ? yabs : -yabs; + return a < b || a > b; + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if operand unordered + /// \retval false else + static bool isunordered(half x, half y) { return isnan(x) || isnan(y); } + + private: + static double erf(double arg) { + if (builtin_isinf(arg)) return (arg < 0.0) ? -1.0 : 1.0; + double x2 = arg * arg, ax2 = 0.147 * x2, + value = std::sqrt( + 1.0 - std::exp(-x2 * (1.2732395447351626861510701069801 + ax2) / + (1.0 + ax2))); + return builtin_signbit(arg) ? -value : value; + } + + static double lgamma(double arg) { + double v = 1.0; + for (; arg < 8.0; ++arg) v *= arg; + double w = 1.0 / (arg * arg); + return (((((((-0.02955065359477124183006535947712 * w + + 0.00641025641025641025641025641026) * + w + + -0.00191752691752691752691752691753) * + w + + 8.4175084175084175084175084175084e-4) * + w + + -5.952380952380952380952380952381e-4) * + w + + 7.9365079365079365079365079365079e-4) * + w + + -0.00277777777777777777777777777778) * + w + + 0.08333333333333333333333333333333) / + arg + + 0.91893853320467274178032973640562 - std::log(v) - arg + + (arg - 0.5) * std::log(arg); + } +}; + +/// Wrapper for unary half-precision functions needing specialization for +/// individual argument types. +/// \tparam T argument type +template +struct unary_specialized { + /// Negation implementation. + /// \param arg value to negate + /// \return negated value + static HALF_CONSTEXPR half negate(half arg) { + return half(binary, arg.data_ ^ 0x8000); + } + + /// Absolute value implementation. + /// \param arg function argument + /// \return absolute value + static half fabs(half arg) { return half(binary, arg.data_ & 0x7FFF); } +}; +template <> +struct unary_specialized { + static HALF_CONSTEXPR expr negate(float arg) { return expr(-arg); } + static expr fabs(float arg) { return expr(std::fabs(arg)); } +}; + +/// Wrapper for binary half-precision functions needing specialization for +/// individual argument types. +/// \tparam T first argument type +/// \tparam U first argument type +template +struct binary_specialized { + /// Minimum implementation. + /// \param x first operand + /// \param y second operand + /// \return minimum value + static expr fmin(float x, float y) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::fmin(x, y)); +#else + if (builtin_isnan(x)) return expr(y); + if (builtin_isnan(y)) return expr(x); + return expr(std::min(x, y)); +#endif + } + + /// Maximum implementation. + /// \param x first operand + /// \param y second operand + /// \return maximum value + static expr fmax(float x, float y) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::fmax(x, y)); +#else + if (builtin_isnan(x)) return expr(y); + if (builtin_isnan(y)) return expr(x); + return expr(std::max(x, y)); +#endif + } +}; +template <> +struct binary_specialized { + static half fmin(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + if (xabs > 0x7C00) return y; + if (yabs > 0x7C00) return x; + return (((xabs == x.data_) ? xabs : -xabs) > + ((yabs == y.data_) ? yabs : -yabs)) + ? y + : x; + } + static half fmax(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + if (xabs > 0x7C00) return y; + if (yabs > 0x7C00) return x; + return (((xabs == x.data_) ? xabs : -xabs) < + ((yabs == y.data_) ? yabs : -yabs)) + ? y + : x; + } +}; + +/// Helper class for half casts. +/// This class template has to be specialized for all valid cast argument to +/// define an appropriate static `cast` member +/// function and a corresponding `type` member denoting its return type. +/// \tparam T destination type +/// \tparam U source type +/// \tparam R rounding mode to use +template +struct half_caster {}; +template +struct half_caster { +#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, + "half_cast from non-arithmetic type unsupported"); +#endif - private: - static double erf(double arg) - { - if(builtin_isinf(arg)) - return (arg<0.0) ? -1.0 : 1.0; - double x2 = arg * arg, ax2 = 0.147 * x2, value = std::sqrt(1.0-std::exp(-x2*(1.2732395447351626861510701069801+ax2)/(1.0+ax2))); - return builtin_signbit(arg) ? -value : value; - } + static half cast(U arg) { return cast_impl(arg, is_float()); }; + + private: + static half cast_impl(U arg, true_type) { + return half(binary, float2half(arg)); + } + static half cast_impl(U arg, false_type) { + return half(binary, int2half(arg)); + } +}; +template +struct half_caster { +#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, + "half_cast to non-arithmetic type unsupported"); +#endif - static double lgamma(double arg) - { - double v = 1.0; - for(; arg<8.0; ++arg) v *= arg; - double w = 1.0 / (arg*arg); - return (((((((-0.02955065359477124183006535947712*w+0.00641025641025641025641025641026)*w+ - -0.00191752691752691752691752691753)*w+8.4175084175084175084175084175084e-4)*w+ - -5.952380952380952380952380952381e-4)*w+7.9365079365079365079365079365079e-4)*w+ - -0.00277777777777777777777777777778)*w+0.08333333333333333333333333333333)/arg + - 0.91893853320467274178032973640562 - std::log(v) - arg + (arg-0.5) * std::log(arg); - } - }; - - /// Wrapper for unary half-precision functions needing specialization for individual argument types. - /// \tparam T argument type - template struct unary_specialized - { - /// Negation implementation. - /// \param arg value to negate - /// \return negated value - static HALF_CONSTEXPR half negate(half arg) { return half(binary, arg.data_^0x8000); } - - /// Absolute value implementation. - /// \param arg function argument - /// \return absolute value - static half fabs(half arg) { return half(binary, arg.data_&0x7FFF); } - }; - template<> struct unary_specialized - { - static HALF_CONSTEXPR expr negate(float arg) { return expr(-arg); } - static expr fabs(float arg) { return expr(std::fabs(arg)); } - }; - - /// Wrapper for binary half-precision functions needing specialization for individual argument types. - /// \tparam T first argument type - /// \tparam U first argument type - template struct binary_specialized - { - /// Minimum implementation. - /// \param x first operand - /// \param y second operand - /// \return minimum value - static expr fmin(float x, float y) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::fmin(x, y)); - #else - if(builtin_isnan(x)) - return expr(y); - if(builtin_isnan(y)) - return expr(x); - return expr(std::min(x, y)); - #endif - } + static T cast(half arg) { return cast_impl(arg, is_float()); } + + private: + static T cast_impl(half arg, true_type) { return half2float(arg.data_); } + static T cast_impl(half arg, false_type) { return half2int(arg.data_); } +}; +template +struct half_caster { +#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, + "half_cast to non-arithmetic type unsupported"); +#endif - /// Maximum implementation. - /// \param x first operand - /// \param y second operand - /// \return maximum value - static expr fmax(float x, float y) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::fmax(x, y)); - #else - if(builtin_isnan(x)) - return expr(y); - if(builtin_isnan(y)) - return expr(x); - return expr(std::max(x, y)); - #endif - } - }; - template<> struct binary_specialized - { - static half fmin(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - if(xabs > 0x7C00) - return y; - if(yabs > 0x7C00) - return x; - return (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs)) ? y : x; - } - static half fmax(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - if(xabs > 0x7C00) - return y; - if(yabs > 0x7C00) - return x; - return (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs)) ? y : x; - } - }; - - /// Helper class for half casts. - /// This class template has to be specialized for all valid cast argument to define an appropriate static `cast` member - /// function and a corresponding `type` member denoting its return type. - /// \tparam T destination type - /// \tparam U source type - /// \tparam R rounding mode to use - template struct half_caster {}; - template struct half_caster - { - #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_arithmetic::value, "half_cast from non-arithmetic type unsupported"); - #endif - - static half cast(U arg) { return cast_impl(arg, is_float()); }; - - private: - static half cast_impl(U arg, true_type) { return half(binary, float2half(arg)); } - static half cast_impl(U arg, false_type) { return half(binary, int2half(arg)); } - }; - template struct half_caster - { - #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_arithmetic::value, "half_cast to non-arithmetic type unsupported"); - #endif - - static T cast(half arg) { return cast_impl(arg, is_float()); } - - private: - static T cast_impl(half arg, true_type) { return half2float(arg.data_); } - static T cast_impl(half arg, false_type) { return half2int(arg.data_); } - }; - template struct half_caster - { - #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_arithmetic::value, "half_cast to non-arithmetic type unsupported"); - #endif - - static T cast(expr arg) { return cast_impl(arg, is_float()); } - - private: - static T cast_impl(float arg, true_type) { return static_cast(arg); } - static T cast_impl(half arg, false_type) { return half2int(arg.data_); } - }; - template struct half_caster - { - static half cast(half arg) { return arg; } - }; - template struct half_caster : half_caster {}; - - /// \name Comparison operators - /// \{ - - /// Comparison for equality. - /// \param x first operand - /// \param y second operand - /// \retval true if operands equal - /// \retval false else - template typename enable::type operator==(T x, U y) { return functions::isequal(x, y); } - - /// Comparison for inequality. - /// \param x first operand - /// \param y second operand - /// \retval true if operands not equal - /// \retval false else - template typename enable::type operator!=(T x, U y) { return functions::isnotequal(x, y); } - - /// Comparison for less than. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x less than \a y - /// \retval false else - template typename enable::type operator<(T x, U y) { return functions::isless(x, y); } - - /// Comparison for greater than. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x greater than \a y - /// \retval false else - template typename enable::type operator>(T x, U y) { return functions::isgreater(x, y); } - - /// Comparison for less equal. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x less equal \a y - /// \retval false else - template typename enable::type operator<=(T x, U y) { return functions::islessequal(x, y); } - - /// Comparison for greater equal. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x greater equal \a y - /// \retval false else - template typename enable::type operator>=(T x, U y) { return functions::isgreaterequal(x, y); } - - /// \} - /// \name Arithmetic operators - /// \{ - - /// Add halfs. - /// \param x left operand - /// \param y right operand - /// \return sum of half expressions - template typename enable::type operator+(T x, U y) { return functions::plus(x, y); } - - /// Subtract halfs. - /// \param x left operand - /// \param y right operand - /// \return difference of half expressions - template typename enable::type operator-(T x, U y) { return functions::minus(x, y); } - - /// Multiply halfs. - /// \param x left operand - /// \param y right operand - /// \return product of half expressions - template typename enable::type operator*(T x, U y) { return functions::multiplies(x, y); } - - /// Divide halfs. - /// \param x left operand - /// \param y right operand - /// \return quotient of half expressions - template typename enable::type operator/(T x, U y) { return functions::divides(x, y); } - - /// Identity. - /// \param arg operand - /// \return uncahnged operand - template HALF_CONSTEXPR typename enable::type operator+(T arg) { return arg; } - - /// Negation. - /// \param arg operand - /// \return negated operand - template HALF_CONSTEXPR typename enable::type operator-(T arg) { return unary_specialized::negate(arg); } - - /// \} - /// \name Input and output - /// \{ - - /// Output operator. - /// \param out output stream to write into - /// \param arg half expression to write - /// \return reference to output stream - template typename enable&,T>::type - operator<<(std::basic_ostream &out, T arg) { return functions::write(out, arg); } - - /// Input operator. - /// \param in input stream to read from - /// \param arg half to read into - /// \return reference to input stream - template std::basic_istream& - operator>>(std::basic_istream &in, half &arg) { return functions::read(in, arg); } - - /// \} - /// \name Basic mathematical operations - /// \{ - - /// Absolute value. - /// \param arg operand - /// \return absolute value of \a arg -// template typename enable::type abs(T arg) { return unary_specialized::fabs(arg); } - inline half abs(half arg) { return unary_specialized::fabs(arg); } - inline expr abs(expr arg) { return unary_specialized::fabs(arg); } - - /// Absolute value. - /// \param arg operand - /// \return absolute value of \a arg -// template typename enable::type fabs(T arg) { return unary_specialized::fabs(arg); } - inline half fabs(half arg) { return unary_specialized::fabs(arg); } - inline expr fabs(expr arg) { return unary_specialized::fabs(arg); } - - /// Remainder of division. - /// \param x first operand - /// \param y second operand - /// \return remainder of floating point division. -// template typename enable::type fmod(T x, U y) { return functions::fmod(x, y); } - inline expr fmod(half x, half y) { return functions::fmod(x, y); } - inline expr fmod(half x, expr y) { return functions::fmod(x, y); } - inline expr fmod(expr x, half y) { return functions::fmod(x, y); } - inline expr fmod(expr x, expr y) { return functions::fmod(x, y); } - - /// Remainder of division. - /// \param x first operand - /// \param y second operand - /// \return remainder of floating point division. -// template typename enable::type remainder(T x, U y) { return functions::remainder(x, y); } - inline expr remainder(half x, half y) { return functions::remainder(x, y); } - inline expr remainder(half x, expr y) { return functions::remainder(x, y); } - inline expr remainder(expr x, half y) { return functions::remainder(x, y); } - inline expr remainder(expr x, expr y) { return functions::remainder(x, y); } - - /// Remainder of division. - /// \param x first operand - /// \param y second operand - /// \param quo address to store some bits of quotient at - /// \return remainder of floating point division. -// template typename enable::type remquo(T x, U y, int *quo) { return functions::remquo(x, y, quo); } - inline expr remquo(half x, half y, int *quo) { return functions::remquo(x, y, quo); } - inline expr remquo(half x, expr y, int *quo) { return functions::remquo(x, y, quo); } - inline expr remquo(expr x, half y, int *quo) { return functions::remquo(x, y, quo); } - inline expr remquo(expr x, expr y, int *quo) { return functions::remquo(x, y, quo); } - - /// Fused multiply add. - /// \param x first operand - /// \param y second operand - /// \param z third operand - /// \return ( \a x * \a y ) + \a z rounded as one operation. -// template typename enable::type fma(T x, U y, V z) { return functions::fma(x, y, z); } - inline expr fma(half x, half y, half z) { return functions::fma(x, y, z); } - inline expr fma(half x, half y, expr z) { return functions::fma(x, y, z); } - inline expr fma(half x, expr y, half z) { return functions::fma(x, y, z); } - inline expr fma(half x, expr y, expr z) { return functions::fma(x, y, z); } - inline expr fma(expr x, half y, half z) { return functions::fma(x, y, z); } - inline expr fma(expr x, half y, expr z) { return functions::fma(x, y, z); } - inline expr fma(expr x, expr y, half z) { return functions::fma(x, y, z); } - inline expr fma(expr x, expr y, expr z) { return functions::fma(x, y, z); } - - /// Maximum of half expressions. - /// \param x first operand - /// \param y second operand - /// \return maximum of operands -// template typename result::type fmax(T x, U y) { return binary_specialized::fmax(x, y); } - inline half fmax(half x, half y) { return binary_specialized::fmax(x, y); } - inline expr fmax(half x, expr y) { return binary_specialized::fmax(x, y); } - inline expr fmax(expr x, half y) { return binary_specialized::fmax(x, y); } - inline expr fmax(expr x, expr y) { return binary_specialized::fmax(x, y); } - - /// Minimum of half expressions. - /// \param x first operand - /// \param y second operand - /// \return minimum of operands -// template typename result::type fmin(T x, U y) { return binary_specialized::fmin(x, y); } - inline half fmin(half x, half y) { return binary_specialized::fmin(x, y); } - inline expr fmin(half x, expr y) { return binary_specialized::fmin(x, y); } - inline expr fmin(expr x, half y) { return binary_specialized::fmin(x, y); } - inline expr fmin(expr x, expr y) { return binary_specialized::fmin(x, y); } - - /// Positive difference. - /// \param x first operand - /// \param y second operand - /// \return \a x - \a y or 0 if difference negative -// template typename enable::type fdim(T x, U y) { return functions::fdim(x, y); } - inline expr fdim(half x, half y) { return functions::fdim(x, y); } - inline expr fdim(half x, expr y) { return functions::fdim(x, y); } - inline expr fdim(expr x, half y) { return functions::fdim(x, y); } - inline expr fdim(expr x, expr y) { return functions::fdim(x, y); } - - /// Get NaN value. - /// \return quiet NaN - inline half nanh(const char*) { return functions::nanh(); } - - /// \} - /// \name Exponential functions - /// \{ - - /// Exponential function. - /// \param arg function argument - /// \return e raised to \a arg -// template typename enable::type exp(T arg) { return functions::exp(arg); } - inline expr exp(half arg) { return functions::exp(arg); } - inline expr exp(expr arg) { return functions::exp(arg); } - - /// Exponential minus one. - /// \param arg function argument - /// \return e raised to \a arg subtracted by 1 -// template typename enable::type expm1(T arg) { return functions::expm1(arg); } - inline expr expm1(half arg) { return functions::expm1(arg); } - inline expr expm1(expr arg) { return functions::expm1(arg); } - - /// Binary exponential. - /// \param arg function argument - /// \return 2 raised to \a arg -// template typename enable::type exp2(T arg) { return functions::exp2(arg); } - inline expr exp2(half arg) { return functions::exp2(arg); } - inline expr exp2(expr arg) { return functions::exp2(arg); } - - /// Natural logorithm. - /// \param arg function argument - /// \return logarithm of \a arg to base e -// template typename enable::type log(T arg) { return functions::log(arg); } - inline expr log(half arg) { return functions::log(arg); } - inline expr log(expr arg) { return functions::log(arg); } - - /// Common logorithm. - /// \param arg function argument - /// \return logarithm of \a arg to base 10 -// template typename enable::type log10(T arg) { return functions::log10(arg); } - inline expr log10(half arg) { return functions::log10(arg); } - inline expr log10(expr arg) { return functions::log10(arg); } - - /// Natural logorithm. - /// \param arg function argument - /// \return logarithm of \a arg plus 1 to base e -// template typename enable::type log1p(T arg) { return functions::log1p(arg); } - inline expr log1p(half arg) { return functions::log1p(arg); } - inline expr log1p(expr arg) { return functions::log1p(arg); } - - /// Binary logorithm. - /// \param arg function argument - /// \return logarithm of \a arg to base 2 -// template typename enable::type log2(T arg) { return functions::log2(arg); } - inline expr log2(half arg) { return functions::log2(arg); } - inline expr log2(expr arg) { return functions::log2(arg); } - - /// \} - /// \name Power functions - /// \{ - - /// Square root. - /// \param arg function argument - /// \return square root of \a arg -// template typename enable::type sqrt(T arg) { return functions::sqrt(arg); } - inline expr sqrt(half arg) { return functions::sqrt(arg); } - inline expr sqrt(expr arg) { return functions::sqrt(arg); } - - /// Cubic root. - /// \param arg function argument - /// \return cubic root of \a arg -// template typename enable::type cbrt(T arg) { return functions::cbrt(arg); } - inline expr cbrt(half arg) { return functions::cbrt(arg); } - inline expr cbrt(expr arg) { return functions::cbrt(arg); } - - /// Hypotenuse function. - /// \param x first argument - /// \param y second argument - /// \return square root of sum of squares without internal over- or underflows -// template typename enable::type hypot(T x, U y) { return functions::hypot(x, y); } - inline expr hypot(half x, half y) { return functions::hypot(x, y); } - inline expr hypot(half x, expr y) { return functions::hypot(x, y); } - inline expr hypot(expr x, half y) { return functions::hypot(x, y); } - inline expr hypot(expr x, expr y) { return functions::hypot(x, y); } - - /// Power function. - /// \param base first argument - /// \param exp second argument - /// \return \a base raised to \a exp -// template typename enable::type pow(T base, U exp) { return functions::pow(base, exp); } - inline expr pow(half base, half exp) { return functions::pow(base, exp); } - inline expr pow(half base, expr exp) { return functions::pow(base, exp); } - inline expr pow(expr base, half exp) { return functions::pow(base, exp); } - inline expr pow(expr base, expr exp) { return functions::pow(base, exp); } - - /// \} - /// \name Trigonometric functions - /// \{ - - /// Sine function. - /// \param arg function argument - /// \return sine value of \a arg -// template typename enable::type sin(T arg) { return functions::sin(arg); } - inline expr sin(half arg) { return functions::sin(arg); } - inline expr sin(expr arg) { return functions::sin(arg); } - - /// Cosine function. - /// \param arg function argument - /// \return cosine value of \a arg -// template typename enable::type cos(T arg) { return functions::cos(arg); } - inline expr cos(half arg) { return functions::cos(arg); } - inline expr cos(expr arg) { return functions::cos(arg); } - - /// Tangent function. - /// \param arg function argument - /// \return tangent value of \a arg -// template typename enable::type tan(T arg) { return functions::tan(arg); } - inline expr tan(half arg) { return functions::tan(arg); } - inline expr tan(expr arg) { return functions::tan(arg); } - - /// Arc sine. - /// \param arg function argument - /// \return arc sine value of \a arg -// template typename enable::type asin(T arg) { return functions::asin(arg); } - inline expr asin(half arg) { return functions::asin(arg); } - inline expr asin(expr arg) { return functions::asin(arg); } - - /// Arc cosine function. - /// \param arg function argument - /// \return arc cosine value of \a arg -// template typename enable::type acos(T arg) { return functions::acos(arg); } - inline expr acos(half arg) { return functions::acos(arg); } - inline expr acos(expr arg) { return functions::acos(arg); } - - /// Arc tangent function. - /// \param arg function argument - /// \return arc tangent value of \a arg -// template typename enable::type atan(T arg) { return functions::atan(arg); } - inline expr atan(half arg) { return functions::atan(arg); } - inline expr atan(expr arg) { return functions::atan(arg); } - - /// Arc tangent function. - /// \param x first argument - /// \param y second argument - /// \return arc tangent value -// template typename enable::type atan2(T x, U y) { return functions::atan2(x, y); } - inline expr atan2(half x, half y) { return functions::atan2(x, y); } - inline expr atan2(half x, expr y) { return functions::atan2(x, y); } - inline expr atan2(expr x, half y) { return functions::atan2(x, y); } - inline expr atan2(expr x, expr y) { return functions::atan2(x, y); } - - /// \} - /// \name Hyperbolic functions - /// \{ - - /// Hyperbolic sine. - /// \param arg function argument - /// \return hyperbolic sine value of \a arg -// template typename enable::type sinh(T arg) { return functions::sinh(arg); } - inline expr sinh(half arg) { return functions::sinh(arg); } - inline expr sinh(expr arg) { return functions::sinh(arg); } - - /// Hyperbolic cosine. - /// \param arg function argument - /// \return hyperbolic cosine value of \a arg -// template typename enable::type cosh(T arg) { return functions::cosh(arg); } - inline expr cosh(half arg) { return functions::cosh(arg); } - inline expr cosh(expr arg) { return functions::cosh(arg); } - - /// Hyperbolic tangent. - /// \param arg function argument - /// \return hyperbolic tangent value of \a arg -// template typename enable::type tanh(T arg) { return functions::tanh(arg); } - inline expr tanh(half arg) { return functions::tanh(arg); } - inline expr tanh(expr arg) { return functions::tanh(arg); } - - /// Hyperbolic area sine. - /// \param arg function argument - /// \return area sine value of \a arg -// template typename enable::type asinh(T arg) { return functions::asinh(arg); } - inline expr asinh(half arg) { return functions::asinh(arg); } - inline expr asinh(expr arg) { return functions::asinh(arg); } - - /// Hyperbolic area cosine. - /// \param arg function argument - /// \return area cosine value of \a arg -// template typename enable::type acosh(T arg) { return functions::acosh(arg); } - inline expr acosh(half arg) { return functions::acosh(arg); } - inline expr acosh(expr arg) { return functions::acosh(arg); } - - /// Hyperbolic area tangent. - /// \param arg function argument - /// \return area tangent value of \a arg -// template typename enable::type atanh(T arg) { return functions::atanh(arg); } - inline expr atanh(half arg) { return functions::atanh(arg); } - inline expr atanh(expr arg) { return functions::atanh(arg); } - - /// \} - /// \name Error and gamma functions - /// \{ - - /// Error function. - /// \param arg function argument - /// \return error function value of \a arg -// template typename enable::type erf(T arg) { return functions::erf(arg); } - inline expr erf(half arg) { return functions::erf(arg); } - inline expr erf(expr arg) { return functions::erf(arg); } - - /// Complementary error function. - /// \param arg function argument - /// \return 1 minus error function value of \a arg -// template typename enable::type erfc(T arg) { return functions::erfc(arg); } - inline expr erfc(half arg) { return functions::erfc(arg); } - inline expr erfc(expr arg) { return functions::erfc(arg); } - - /// Natural logarithm of gamma function. - /// \param arg function argument - /// \return natural logarith of gamma function for \a arg -// template typename enable::type lgamma(T arg) { return functions::lgamma(arg); } - inline expr lgamma(half arg) { return functions::lgamma(arg); } - inline expr lgamma(expr arg) { return functions::lgamma(arg); } - - /// Gamma function. - /// \param arg function argument - /// \return gamma function value of \a arg -// template typename enable::type tgamma(T arg) { return functions::tgamma(arg); } - inline expr tgamma(half arg) { return functions::tgamma(arg); } - inline expr tgamma(expr arg) { return functions::tgamma(arg); } - - /// \} - /// \name Rounding - /// \{ - - /// Nearest integer not less than half value. - /// \param arg half to round - /// \return nearest integer not less than \a arg -// template typename enable::type ceil(T arg) { return functions::ceil(arg); } - inline half ceil(half arg) { return functions::ceil(arg); } - inline half ceil(expr arg) { return functions::ceil(arg); } - - /// Nearest integer not greater than half value. - /// \param arg half to round - /// \return nearest integer not greater than \a arg -// template typename enable::type floor(T arg) { return functions::floor(arg); } - inline half floor(half arg) { return functions::floor(arg); } - inline half floor(expr arg) { return functions::floor(arg); } - - /// Nearest integer not greater in magnitude than half value. - /// \param arg half to round - /// \return nearest integer not greater in magnitude than \a arg -// template typename enable::type trunc(T arg) { return functions::trunc(arg); } - inline half trunc(half arg) { return functions::trunc(arg); } - inline half trunc(expr arg) { return functions::trunc(arg); } - - /// Nearest integer. - /// \param arg half to round - /// \return nearest integer, rounded away from zero in half-way cases -// template typename enable::type round(T arg) { return functions::round(arg); } - inline half round(half arg) { return functions::round(arg); } - inline half round(expr arg) { return functions::round(arg); } - - /// Nearest integer. - /// \param arg half to round - /// \return nearest integer, rounded away from zero in half-way cases -// template typename enable::type lround(T arg) { return functions::lround(arg); } - inline long lround(half arg) { return functions::lround(arg); } - inline long lround(expr arg) { return functions::lround(arg); } - - /// Nearest integer using half's internal rounding mode. - /// \param arg half expression to round - /// \return nearest integer using default rounding mode -// template typename enable::type nearbyint(T arg) { return functions::nearbyint(arg); } - inline half nearbyint(half arg) { return functions::rint(arg); } - inline half nearbyint(expr arg) { return functions::rint(arg); } - - /// Nearest integer using half's internal rounding mode. - /// \param arg half expression to round - /// \return nearest integer using default rounding mode -// template typename enable::type rint(T arg) { return functions::rint(arg); } - inline half rint(half arg) { return functions::rint(arg); } - inline half rint(expr arg) { return functions::rint(arg); } - - /// Nearest integer using half's internal rounding mode. - /// \param arg half expression to round - /// \return nearest integer using default rounding mode -// template typename enable::type lrint(T arg) { return functions::lrint(arg); } - inline long lrint(half arg) { return functions::lrint(arg); } - inline long lrint(expr arg) { return functions::lrint(arg); } - #if HALF_ENABLE_CPP11_LONG_LONG - /// Nearest integer. - /// \param arg half to round - /// \return nearest integer, rounded away from zero in half-way cases -// template typename enable::type llround(T arg) { return functions::llround(arg); } - inline long long llround(half arg) { return functions::llround(arg); } - inline long long llround(expr arg) { return functions::llround(arg); } - - /// Nearest integer using half's internal rounding mode. - /// \param arg half expression to round - /// \return nearest integer using default rounding mode -// template typename enable::type llrint(T arg) { return functions::llrint(arg); } - inline long long llrint(half arg) { return functions::llrint(arg); } - inline long long llrint(expr arg) { return functions::llrint(arg); } - #endif - - /// \} - /// \name Floating point manipulation - /// \{ - - /// Decompress floating point number. - /// \param arg number to decompress - /// \param exp address to store exponent at - /// \return significant in range [0.5, 1) -// template typename enable::type frexp(T arg, int *exp) { return functions::frexp(arg, exp); } - inline half frexp(half arg, int *exp) { return functions::frexp(arg, exp); } - inline half frexp(expr arg, int *exp) { return functions::frexp(arg, exp); } - - /// Multiply by power of two. - /// \param arg number to modify - /// \param exp power of two to multiply with - /// \return \a arg multplied by 2 raised to \a exp -// template typename enable::type ldexp(T arg, int exp) { return functions::scalbln(arg, exp); } - inline half ldexp(half arg, int exp) { return functions::scalbln(arg, exp); } - inline half ldexp(expr arg, int exp) { return functions::scalbln(arg, exp); } - - /// Extract integer and fractional parts. - /// \param arg number to decompress - /// \param iptr address to store integer part at - /// \return fractional part -// template typename enable::type modf(T arg, half *iptr) { return functions::modf(arg, iptr); } - inline half modf(half arg, half *iptr) { return functions::modf(arg, iptr); } - inline half modf(expr arg, half *iptr) { return functions::modf(arg, iptr); } - - /// Multiply by power of two. - /// \param arg number to modify - /// \param exp power of two to multiply with - /// \return \a arg multplied by 2 raised to \a exp -// template typename enable::type scalbn(T arg, int exp) { return functions::scalbln(arg, exp); } - inline half scalbn(half arg, int exp) { return functions::scalbln(arg, exp); } - inline half scalbn(expr arg, int exp) { return functions::scalbln(arg, exp); } - - /// Multiply by power of two. - /// \param arg number to modify - /// \param exp power of two to multiply with - /// \return \a arg multplied by 2 raised to \a exp -// template typename enable::type scalbln(T arg, long exp) { return functions::scalbln(arg, exp); } - inline half scalbln(half arg, long exp) { return functions::scalbln(arg, exp); } - inline half scalbln(expr arg, long exp) { return functions::scalbln(arg, exp); } - - /// Extract exponent. - /// \param arg number to query - /// \return floating point exponent - /// \retval FP_ILOGB0 for zero - /// \retval FP_ILOGBNAN for NaN - /// \retval MAX_INT for infinity -// template typename enable::type ilogb(T arg) { return functions::ilogb(arg); } - inline int ilogb(half arg) { return functions::ilogb(arg); } - inline int ilogb(expr arg) { return functions::ilogb(arg); } - - /// Extract exponent. - /// \param arg number to query - /// \return floating point exponent -// template typename enable::type logb(T arg) { return functions::logb(arg); } - inline half logb(half arg) { return functions::logb(arg); } - inline half logb(expr arg) { return functions::logb(arg); } - - /// Next representable value. - /// \param from value to compute next representable value for - /// \param to direction towards which to compute next value - /// \return next representable value after \a from in direction towards \a to -// template typename enable::type nextafter(T from, U to) { return functions::nextafter(from, to); } - inline half nextafter(half from, half to) { return functions::nextafter(from, to); } - inline half nextafter(half from, expr to) { return functions::nextafter(from, to); } - inline half nextafter(expr from, half to) { return functions::nextafter(from, to); } - inline half nextafter(expr from, expr to) { return functions::nextafter(from, to); } - - /// Next representable value. - /// \param from value to compute next representable value for - /// \param to direction towards which to compute next value - /// \return next representable value after \a from in direction towards \a to -// template typename enable::type nexttoward(T from, long double to) { return functions::nexttoward(from, to); } - inline half nexttoward(half from, long double to) { return functions::nexttoward(from, to); } - inline half nexttoward(expr from, long double to) { return functions::nexttoward(from, to); } - - /// Take sign. - /// \param x value to change sign for - /// \param y value to take sign from - /// \return value equal to \a x in magnitude and to \a y in sign -// template typename enable::type copysign(T x, U y) { return functions::copysign(x, y); } - inline half copysign(half x, half y) { return functions::copysign(x, y); } - inline half copysign(half x, expr y) { return functions::copysign(x, y); } - inline half copysign(expr x, half y) { return functions::copysign(x, y); } - inline half copysign(expr x, expr y) { return functions::copysign(x, y); } - - /// \} - /// \name Floating point classification - /// \{ - - - /// Classify floating point value. - /// \param arg number to classify - /// \retval FP_ZERO for positive and negative zero - /// \retval FP_SUBNORMAL for subnormal numbers - /// \retval FP_INFINITY for positive and negative infinity - /// \retval FP_NAN for NaNs - /// \retval FP_NORMAL for all other (normal) values -// template typename enable::type fpclassify(T arg) { return functions::fpclassify(arg); } - inline int fpclassify(half arg) { return functions::fpclassify(arg); } - inline int fpclassify(expr arg) { return functions::fpclassify(arg); } - - /// Check if finite number. - /// \param arg number to check - /// \retval true if neither infinity nor NaN - /// \retval false else -// template typename enable::type isfinite(T arg) { return functions::isfinite(arg); } - inline bool isfinite(half arg) { return functions::isfinite(arg); } - inline bool isfinite(expr arg) { return functions::isfinite(arg); } - - /// Check for infinity. - /// \param arg number to check - /// \retval true for positive or negative infinity - /// \retval false else -// template typename enable::type isinf(T arg) { return functions::isinf(arg); } - inline bool isinf(half arg) { return functions::isinf(arg); } - inline bool isinf(expr arg) { return functions::isinf(arg); } - - /// Check for NaN. - /// \param arg number to check - /// \retval true for NaNs - /// \retval false else -// template typename enable::type isnan(T arg) { return functions::isnan(arg); } - inline bool isnan(half arg) { return functions::isnan(arg); } - inline bool isnan(expr arg) { return functions::isnan(arg); } - - /// Check if normal number. - /// \param arg number to check - /// \retval true if normal number - /// \retval false if either subnormal, zero, infinity or NaN -// template typename enable::type isnormal(T arg) { return functions::isnormal(arg); } - inline bool isnormal(half arg) { return functions::isnormal(arg); } - inline bool isnormal(expr arg) { return functions::isnormal(arg); } - - /// Check sign. - /// \param arg number to check - /// \retval true for negative number - /// \retval false for positive number -// template typename enable::type signbit(T arg) { return functions::signbit(arg); } - inline bool signbit(half arg) { return functions::signbit(arg); } - inline bool signbit(expr arg) { return functions::signbit(arg); } - - /// \} - /// \name Comparison - /// \{ - - /// Comparison for greater than. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x greater than \a y - /// \retval false else -// template typename enable::type isgreater(T x, U y) { return functions::isgreater(x, y); } - inline bool isgreater(half x, half y) { return functions::isgreater(x, y); } - inline bool isgreater(half x, expr y) { return functions::isgreater(x, y); } - inline bool isgreater(expr x, half y) { return functions::isgreater(x, y); } - inline bool isgreater(expr x, expr y) { return functions::isgreater(x, y); } - - /// Comparison for greater equal. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x greater equal \a y - /// \retval false else -// template typename enable::type isgreaterequal(T x, U y) { return functions::isgreaterequal(x, y); } - inline bool isgreaterequal(half x, half y) { return functions::isgreaterequal(x, y); } - inline bool isgreaterequal(half x, expr y) { return functions::isgreaterequal(x, y); } - inline bool isgreaterequal(expr x, half y) { return functions::isgreaterequal(x, y); } - inline bool isgreaterequal(expr x, expr y) { return functions::isgreaterequal(x, y); } - - /// Comparison for less than. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x less than \a y - /// \retval false else -// template typename enable::type isless(T x, U y) { return functions::isless(x, y); } - inline bool isless(half x, half y) { return functions::isless(x, y); } - inline bool isless(half x, expr y) { return functions::isless(x, y); } - inline bool isless(expr x, half y) { return functions::isless(x, y); } - inline bool isless(expr x, expr y) { return functions::isless(x, y); } - - /// Comparison for less equal. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x less equal \a y - /// \retval false else -// template typename enable::type islessequal(T x, U y) { return functions::islessequal(x, y); } - inline bool islessequal(half x, half y) { return functions::islessequal(x, y); } - inline bool islessequal(half x, expr y) { return functions::islessequal(x, y); } - inline bool islessequal(expr x, half y) { return functions::islessequal(x, y); } - inline bool islessequal(expr x, expr y) { return functions::islessequal(x, y); } - - /// Comarison for less or greater. - /// \param x first operand - /// \param y second operand - /// \retval true if either less or greater - /// \retval false else -// template typename enable::type islessgreater(T x, U y) { return functions::islessgreater(x, y); } - inline bool islessgreater(half x, half y) { return functions::islessgreater(x, y); } - inline bool islessgreater(half x, expr y) { return functions::islessgreater(x, y); } - inline bool islessgreater(expr x, half y) { return functions::islessgreater(x, y); } - inline bool islessgreater(expr x, expr y) { return functions::islessgreater(x, y); } - - /// Check if unordered. - /// \param x first operand - /// \param y second operand - /// \retval true if unordered (one or two NaN operands) - /// \retval false else -// template typename enable::type isunordered(T x, U y) { return functions::isunordered(x, y); } - inline bool isunordered(half x, half y) { return functions::isunordered(x, y); } - inline bool isunordered(half x, expr y) { return functions::isunordered(x, y); } - inline bool isunordered(expr x, half y) { return functions::isunordered(x, y); } - inline bool isunordered(expr x, expr y) { return functions::isunordered(x, y); } - - /// \name Casting - /// \{ - - /// Cast to or from half-precision floating point number. - /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted - /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do. - /// It uses the default rounding mode. - /// - /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types - /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler - /// error and casting between [half](\ref half_float::half)s is just a no-op. - /// \tparam T destination type (half or built-in arithmetic type) - /// \tparam U source type (half or built-in arithmetic type) - /// \param arg value to cast - /// \return \a arg converted to destination type - template T half_cast(U arg) { return half_caster::cast(arg); } - - /// Cast to or from half-precision floating point number. - /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted - /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do. - /// - /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types - /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler - /// error and casting between [half](\ref half_float::half)s is just a no-op. - /// \tparam T destination type (half or built-in arithmetic type) - /// \tparam R rounding mode to use. - /// \tparam U source type (half or built-in arithmetic type) - /// \param arg value to cast - /// \return \a arg converted to destination type - template T half_cast(U arg) { return half_caster::cast(arg); } - /// \} - } - - using detail::operator==; - using detail::operator!=; - using detail::operator<; - using detail::operator>; - using detail::operator<=; - using detail::operator>=; - using detail::operator+; - using detail::operator-; - using detail::operator*; - using detail::operator/; - using detail::operator<<; - using detail::operator>>; - - using detail::abs; - using detail::fabs; - using detail::fmod; - using detail::remainder; - using detail::remquo; - using detail::fma; - using detail::fmax; - using detail::fmin; - using detail::fdim; - using detail::nanh; - using detail::exp; - using detail::expm1; - using detail::exp2; - using detail::log; - using detail::log10; - using detail::log1p; - using detail::log2; - using detail::sqrt; - using detail::cbrt; - using detail::hypot; - using detail::pow; - using detail::sin; - using detail::cos; - using detail::tan; - using detail::asin; - using detail::acos; - using detail::atan; - using detail::atan2; - using detail::sinh; - using detail::cosh; - using detail::tanh; - using detail::asinh; - using detail::acosh; - using detail::atanh; - using detail::erf; - using detail::erfc; - using detail::lgamma; - using detail::tgamma; - using detail::ceil; - using detail::floor; - using detail::trunc; - using detail::round; - using detail::lround; - using detail::nearbyint; - using detail::rint; - using detail::lrint; -#if HALF_ENABLE_CPP11_LONG_LONG - using detail::llround; - using detail::llrint; -#endif - using detail::frexp; - using detail::ldexp; - using detail::modf; - using detail::scalbn; - using detail::scalbln; - using detail::ilogb; - using detail::logb; - using detail::nextafter; - using detail::nexttoward; - using detail::copysign; - using detail::fpclassify; - using detail::isfinite; - using detail::isinf; - using detail::isnan; - using detail::isnormal; - using detail::signbit; - using detail::isgreater; - using detail::isgreaterequal; - using detail::isless; - using detail::islessequal; - using detail::islessgreater; - using detail::isunordered; - - using detail::half_cast; + static T cast(expr arg) { return cast_impl(arg, is_float()); } + + private: + static T cast_impl(float arg, true_type) { return static_cast(arg); } + static T cast_impl(half arg, false_type) { return half2int(arg.data_); } +}; +template +struct half_caster { + static half cast(half arg) { return arg; } +}; +template +struct half_caster : half_caster {}; + +/// \name Comparison operators +/// \{ + +/// Comparison for equality. +/// \param x first operand +/// \param y second operand +/// \retval true if operands equal +/// \retval false else +template +typename enable::type operator==(T x, U y) { + return functions::isequal(x, y); } +/// Comparison for inequality. +/// \param x first operand +/// \param y second operand +/// \retval true if operands not equal +/// \retval false else +template +typename enable::type operator!=(T x, U y) { + return functions::isnotequal(x, y); +} -/// Extensions to the C++ standard library. -namespace std -{ - /// Numeric limits for half-precision floats. - /// Because of the underlying single-precision implementation of many operations, it inherits some properties from - /// `std::numeric_limits`. - template<> class numeric_limits : public numeric_limits - { - public: - /// Supports signed values. - static HALF_CONSTEXPR_CONST bool is_signed = true; - - /// Is not exact. - static HALF_CONSTEXPR_CONST bool is_exact = false; +/// Comparison for less than. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x less than \a y +/// \retval false else +template +typename enable::type operator<(T x, U y) { + return functions::isless(x, y); +} - /// Doesn't provide modulo arithmetic. - static HALF_CONSTEXPR_CONST bool is_modulo = false; +/// Comparison for greater than. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x greater than \a y +/// \retval false else +template +typename enable::type operator>(T x, U y) { + return functions::isgreater(x, y); +} - /// IEEE conformant. - static HALF_CONSTEXPR_CONST bool is_iec559 = true; +/// Comparison for less equal. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x less equal \a y +/// \retval false else +template +typename enable::type operator<=(T x, U y) { + return functions::islessequal(x, y); +} - /// Supports infinity. - static HALF_CONSTEXPR_CONST bool has_infinity = true; +/// Comparison for greater equal. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x greater equal \a y +/// \retval false else +template +typename enable::type operator>=(T x, U y) { + return functions::isgreaterequal(x, y); +} - /// Supports quiet NaNs. - static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true; +/// \} +/// \name Arithmetic operators +/// \{ + +/// Add halfs. +/// \param x left operand +/// \param y right operand +/// \return sum of half expressions +template +typename enable::type operator+(T x, U y) { + return functions::plus(x, y); +} - /// Supports subnormal values. - static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present; +/// Subtract halfs. +/// \param x left operand +/// \param y right operand +/// \return difference of half expressions +template +typename enable::type operator-(T x, U y) { + return functions::minus(x, y); +} - /// Rounding mode. - /// Due to the mix of internal single-precision computations (using the rounding mode of the underlying - /// single-precision implementation) with the rounding mode of the single-to-half conversions, the actual rounding - /// mode might be `std::round_indeterminate` if the default half-precision rounding mode doesn't match the - /// single-precision rounding mode. - static HALF_CONSTEXPR_CONST float_round_style round_style = (std::numeric_limits::round_style== - half_float::half::round_style) ? half_float::half::round_style : round_indeterminate; +/// Multiply halfs. +/// \param x left operand +/// \param y right operand +/// \return product of half expressions +template +typename enable::type operator*(T x, U y) { + return functions::multiplies(x, y); +} - /// Significant digits. - static HALF_CONSTEXPR_CONST int digits = 11; +/// Divide halfs. +/// \param x left operand +/// \param y right operand +/// \return quotient of half expressions +template +typename enable::type operator/(T x, U y) { + return functions::divides(x, y); +} - /// Significant decimal digits. - static HALF_CONSTEXPR_CONST int digits10 = 3; +/// Identity. +/// \param arg operand +/// \return uncahnged operand +template +HALF_CONSTEXPR typename enable::type operator+(T arg) { + return arg; +} - /// Required decimal digits to represent all possible values. - static HALF_CONSTEXPR_CONST int max_digits10 = 5; +/// Negation. +/// \param arg operand +/// \return negated operand +template +HALF_CONSTEXPR typename enable::type operator-(T arg) { + return unary_specialized::negate(arg); +} - /// Number base. - static HALF_CONSTEXPR_CONST int radix = 2; +/// \} +/// \name Input and output +/// \{ + +/// Output operator. +/// \param out output stream to write into +/// \param arg half expression to write +/// \return reference to output stream +template +typename enable &, T>::type operator<<( + std::basic_ostream &out, T arg) { + return functions::write(out, arg); +} - /// One more than smallest exponent. - static HALF_CONSTEXPR_CONST int min_exponent = -13; +/// Input operator. +/// \param in input stream to read from +/// \param arg half to read into +/// \return reference to input stream +template +std::basic_istream &operator>>( + std::basic_istream &in, half &arg) { + return functions::read(in, arg); +} - /// Smallest normalized representable power of 10. - static HALF_CONSTEXPR_CONST int min_exponent10 = -4; +/// \} +/// \name Basic mathematical operations +/// \{ + +/// Absolute value. +/// \param arg operand +/// \return absolute value of \a arg +// template typename enable::type abs(T arg) { +//return unary_specialized::fabs(arg); } +inline half abs(half arg) { return unary_specialized::fabs(arg); } +inline expr abs(expr arg) { return unary_specialized::fabs(arg); } + +/// Absolute value. +/// \param arg operand +/// \return absolute value of \a arg +// template typename enable::type fabs(T arg) { +//return unary_specialized::fabs(arg); } +inline half fabs(half arg) { return unary_specialized::fabs(arg); } +inline expr fabs(expr arg) { return unary_specialized::fabs(arg); } + +/// Remainder of division. +/// \param x first operand +/// \param y second operand +/// \return remainder of floating point division. +// template typename enable::type +//fmod(T x, U y) { return functions::fmod(x, y); } +inline expr fmod(half x, half y) { return functions::fmod(x, y); } +inline expr fmod(half x, expr y) { return functions::fmod(x, y); } +inline expr fmod(expr x, half y) { return functions::fmod(x, y); } +inline expr fmod(expr x, expr y) { return functions::fmod(x, y); } + +/// Remainder of division. +/// \param x first operand +/// \param y second operand +/// \return remainder of floating point division. +// template typename enable::type +//remainder(T x, U y) { return functions::remainder(x, y); } +inline expr remainder(half x, half y) { return functions::remainder(x, y); } +inline expr remainder(half x, expr y) { return functions::remainder(x, y); } +inline expr remainder(expr x, half y) { return functions::remainder(x, y); } +inline expr remainder(expr x, expr y) { return functions::remainder(x, y); } + +/// Remainder of division. +/// \param x first operand +/// \param y second operand +/// \param quo address to store some bits of quotient at +/// \return remainder of floating point division. +// template typename enable::type +//remquo(T x, U y, int *quo) { return functions::remquo(x, y, quo); } +inline expr remquo(half x, half y, int *quo) { + return functions::remquo(x, y, quo); +} +inline expr remquo(half x, expr y, int *quo) { + return functions::remquo(x, y, quo); +} +inline expr remquo(expr x, half y, int *quo) { + return functions::remquo(x, y, quo); +} +inline expr remquo(expr x, expr y, int *quo) { + return functions::remquo(x, y, quo); +} - /// One more than largest exponent - static HALF_CONSTEXPR_CONST int max_exponent = 16; +/// Fused multiply add. +/// \param x first operand +/// \param y second operand +/// \param z third operand +/// \return ( \a x * \a y ) + \a z rounded as one operation. +// template typename +//enable::type fma(T x, U y, V z) { return functions::fma(x, y, z); +//} +inline expr fma(half x, half y, half z) { return functions::fma(x, y, z); } +inline expr fma(half x, half y, expr z) { return functions::fma(x, y, z); } +inline expr fma(half x, expr y, half z) { return functions::fma(x, y, z); } +inline expr fma(half x, expr y, expr z) { return functions::fma(x, y, z); } +inline expr fma(expr x, half y, half z) { return functions::fma(x, y, z); } +inline expr fma(expr x, half y, expr z) { return functions::fma(x, y, z); } +inline expr fma(expr x, expr y, half z) { return functions::fma(x, y, z); } +inline expr fma(expr x, expr y, expr z) { return functions::fma(x, y, z); } + +/// Maximum of half expressions. +/// \param x first operand +/// \param y second operand +/// \return maximum of operands +// template typename result::type fmax(T +//x, U y) { return binary_specialized::fmax(x, y); } +inline half fmax(half x, half y) { + return binary_specialized::fmax(x, y); +} +inline expr fmax(half x, expr y) { + return binary_specialized::fmax(x, y); +} +inline expr fmax(expr x, half y) { + return binary_specialized::fmax(x, y); +} +inline expr fmax(expr x, expr y) { + return binary_specialized::fmax(x, y); +} - /// Largest finitely representable power of 10. - static HALF_CONSTEXPR_CONST int max_exponent10 = 4; +/// Minimum of half expressions. +/// \param x first operand +/// \param y second operand +/// \return minimum of operands +// template typename result::type fmin(T +//x, U y) { return binary_specialized::fmin(x, y); } +inline half fmin(half x, half y) { + return binary_specialized::fmin(x, y); +} +inline expr fmin(half x, expr y) { + return binary_specialized::fmin(x, y); +} +inline expr fmin(expr x, half y) { + return binary_specialized::fmin(x, y); +} +inline expr fmin(expr x, expr y) { + return binary_specialized::fmin(x, y); +} - /// Smallest positive normal value. - static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0400); } +/// Positive difference. +/// \param x first operand +/// \param y second operand +/// \return \a x - \a y or 0 if difference negative +// template typename enable::type +//fdim(T x, U y) { return functions::fdim(x, y); } +inline expr fdim(half x, half y) { return functions::fdim(x, y); } +inline expr fdim(half x, expr y) { return functions::fdim(x, y); } +inline expr fdim(expr x, half y) { return functions::fdim(x, y); } +inline expr fdim(expr x, expr y) { return functions::fdim(x, y); } + +/// Get NaN value. +/// \return quiet NaN +inline half nanh(const char *) { return functions::nanh(); } + +/// \} +/// \name Exponential functions +/// \{ + +/// Exponential function. +/// \param arg function argument +/// \return e raised to \a arg +// template typename enable::type exp(T arg) { +//return functions::exp(arg); } +inline expr exp(half arg) { return functions::exp(arg); } +inline expr exp(expr arg) { return functions::exp(arg); } + +/// Exponential minus one. +/// \param arg function argument +/// \return e raised to \a arg subtracted by 1 +// template typename enable::type expm1(T arg) { +//return functions::expm1(arg); } +inline expr expm1(half arg) { return functions::expm1(arg); } +inline expr expm1(expr arg) { return functions::expm1(arg); } + +/// Binary exponential. +/// \param arg function argument +/// \return 2 raised to \a arg +// template typename enable::type exp2(T arg) { +//return functions::exp2(arg); } +inline expr exp2(half arg) { return functions::exp2(arg); } +inline expr exp2(expr arg) { return functions::exp2(arg); } + +/// Natural logorithm. +/// \param arg function argument +/// \return logarithm of \a arg to base e +// template typename enable::type log(T arg) { +//return functions::log(arg); } +inline expr log(half arg) { return functions::log(arg); } +inline expr log(expr arg) { return functions::log(arg); } + +/// Common logorithm. +/// \param arg function argument +/// \return logarithm of \a arg to base 10 +// template typename enable::type log10(T arg) { +//return functions::log10(arg); } +inline expr log10(half arg) { return functions::log10(arg); } +inline expr log10(expr arg) { return functions::log10(arg); } + +/// Natural logorithm. +/// \param arg function argument +/// \return logarithm of \a arg plus 1 to base e +// template typename enable::type log1p(T arg) { +//return functions::log1p(arg); } +inline expr log1p(half arg) { return functions::log1p(arg); } +inline expr log1p(expr arg) { return functions::log1p(arg); } + +/// Binary logorithm. +/// \param arg function argument +/// \return logarithm of \a arg to base 2 +// template typename enable::type log2(T arg) { +//return functions::log2(arg); } +inline expr log2(half arg) { return functions::log2(arg); } +inline expr log2(expr arg) { return functions::log2(arg); } + +/// \} +/// \name Power functions +/// \{ + +/// Square root. +/// \param arg function argument +/// \return square root of \a arg +// template typename enable::type sqrt(T arg) { +//return functions::sqrt(arg); } +inline expr sqrt(half arg) { return functions::sqrt(arg); } +inline expr sqrt(expr arg) { return functions::sqrt(arg); } + +/// Cubic root. +/// \param arg function argument +/// \return cubic root of \a arg +// template typename enable::type cbrt(T arg) { +//return functions::cbrt(arg); } +inline expr cbrt(half arg) { return functions::cbrt(arg); } +inline expr cbrt(expr arg) { return functions::cbrt(arg); } + +/// Hypotenuse function. +/// \param x first argument +/// \param y second argument +/// \return square root of sum of squares without internal over- or underflows +// template typename enable::type +//hypot(T x, U y) { return functions::hypot(x, y); } +inline expr hypot(half x, half y) { return functions::hypot(x, y); } +inline expr hypot(half x, expr y) { return functions::hypot(x, y); } +inline expr hypot(expr x, half y) { return functions::hypot(x, y); } +inline expr hypot(expr x, expr y) { return functions::hypot(x, y); } + +/// Power function. +/// \param base first argument +/// \param exp second argument +/// \return \a base raised to \a exp +// template typename enable::type +//pow(T base, U exp) { return functions::pow(base, exp); } +inline expr pow(half base, half exp) { return functions::pow(base, exp); } +inline expr pow(half base, expr exp) { return functions::pow(base, exp); } +inline expr pow(expr base, half exp) { return functions::pow(base, exp); } +inline expr pow(expr base, expr exp) { return functions::pow(base, exp); } + +/// \} +/// \name Trigonometric functions +/// \{ + +/// Sine function. +/// \param arg function argument +/// \return sine value of \a arg +// template typename enable::type sin(T arg) { +//return functions::sin(arg); } +inline expr sin(half arg) { return functions::sin(arg); } +inline expr sin(expr arg) { return functions::sin(arg); } + +/// Cosine function. +/// \param arg function argument +/// \return cosine value of \a arg +// template typename enable::type cos(T arg) { +//return functions::cos(arg); } +inline expr cos(half arg) { return functions::cos(arg); } +inline expr cos(expr arg) { return functions::cos(arg); } + +/// Tangent function. +/// \param arg function argument +/// \return tangent value of \a arg +// template typename enable::type tan(T arg) { +//return functions::tan(arg); } +inline expr tan(half arg) { return functions::tan(arg); } +inline expr tan(expr arg) { return functions::tan(arg); } + +/// Arc sine. +/// \param arg function argument +/// \return arc sine value of \a arg +// template typename enable::type asin(T arg) { +//return functions::asin(arg); } +inline expr asin(half arg) { return functions::asin(arg); } +inline expr asin(expr arg) { return functions::asin(arg); } + +/// Arc cosine function. +/// \param arg function argument +/// \return arc cosine value of \a arg +// template typename enable::type acos(T arg) { +//return functions::acos(arg); } +inline expr acos(half arg) { return functions::acos(arg); } +inline expr acos(expr arg) { return functions::acos(arg); } + +/// Arc tangent function. +/// \param arg function argument +/// \return arc tangent value of \a arg +// template typename enable::type atan(T arg) { +//return functions::atan(arg); } +inline expr atan(half arg) { return functions::atan(arg); } +inline expr atan(expr arg) { return functions::atan(arg); } + +/// Arc tangent function. +/// \param x first argument +/// \param y second argument +/// \return arc tangent value +// template typename enable::type +//atan2(T x, U y) { return functions::atan2(x, y); } +inline expr atan2(half x, half y) { return functions::atan2(x, y); } +inline expr atan2(half x, expr y) { return functions::atan2(x, y); } +inline expr atan2(expr x, half y) { return functions::atan2(x, y); } +inline expr atan2(expr x, expr y) { return functions::atan2(x, y); } + +/// \} +/// \name Hyperbolic functions +/// \{ + +/// Hyperbolic sine. +/// \param arg function argument +/// \return hyperbolic sine value of \a arg +// template typename enable::type sinh(T arg) { +//return functions::sinh(arg); } +inline expr sinh(half arg) { return functions::sinh(arg); } +inline expr sinh(expr arg) { return functions::sinh(arg); } + +/// Hyperbolic cosine. +/// \param arg function argument +/// \return hyperbolic cosine value of \a arg +// template typename enable::type cosh(T arg) { +//return functions::cosh(arg); } +inline expr cosh(half arg) { return functions::cosh(arg); } +inline expr cosh(expr arg) { return functions::cosh(arg); } + +/// Hyperbolic tangent. +/// \param arg function argument +/// \return hyperbolic tangent value of \a arg +// template typename enable::type tanh(T arg) { +//return functions::tanh(arg); } +inline expr tanh(half arg) { return functions::tanh(arg); } +inline expr tanh(expr arg) { return functions::tanh(arg); } + +/// Hyperbolic area sine. +/// \param arg function argument +/// \return area sine value of \a arg +// template typename enable::type asinh(T arg) { +//return functions::asinh(arg); } +inline expr asinh(half arg) { return functions::asinh(arg); } +inline expr asinh(expr arg) { return functions::asinh(arg); } + +/// Hyperbolic area cosine. +/// \param arg function argument +/// \return area cosine value of \a arg +// template typename enable::type acosh(T arg) { +//return functions::acosh(arg); } +inline expr acosh(half arg) { return functions::acosh(arg); } +inline expr acosh(expr arg) { return functions::acosh(arg); } + +/// Hyperbolic area tangent. +/// \param arg function argument +/// \return area tangent value of \a arg +// template typename enable::type atanh(T arg) { +//return functions::atanh(arg); } +inline expr atanh(half arg) { return functions::atanh(arg); } +inline expr atanh(expr arg) { return functions::atanh(arg); } + +/// \} +/// \name Error and gamma functions +/// \{ + +/// Error function. +/// \param arg function argument +/// \return error function value of \a arg +// template typename enable::type erf(T arg) { +//return functions::erf(arg); } +inline expr erf(half arg) { return functions::erf(arg); } +inline expr erf(expr arg) { return functions::erf(arg); } + +/// Complementary error function. +/// \param arg function argument +/// \return 1 minus error function value of \a arg +// template typename enable::type erfc(T arg) { +//return functions::erfc(arg); } +inline expr erfc(half arg) { return functions::erfc(arg); } +inline expr erfc(expr arg) { return functions::erfc(arg); } + +/// Natural logarithm of gamma function. +/// \param arg function argument +/// \return natural logarith of gamma function for \a arg +// template typename enable::type lgamma(T arg) { +//return functions::lgamma(arg); } +inline expr lgamma(half arg) { return functions::lgamma(arg); } +inline expr lgamma(expr arg) { return functions::lgamma(arg); } + +/// Gamma function. +/// \param arg function argument +/// \return gamma function value of \a arg +// template typename enable::type tgamma(T arg) { +//return functions::tgamma(arg); } +inline expr tgamma(half arg) { return functions::tgamma(arg); } +inline expr tgamma(expr arg) { return functions::tgamma(arg); } + +/// \} +/// \name Rounding +/// \{ + +/// Nearest integer not less than half value. +/// \param arg half to round +/// \return nearest integer not less than \a arg +// template typename enable::type ceil(T arg) { +//return functions::ceil(arg); } +inline half ceil(half arg) { return functions::ceil(arg); } +inline half ceil(expr arg) { return functions::ceil(arg); } + +/// Nearest integer not greater than half value. +/// \param arg half to round +/// \return nearest integer not greater than \a arg +// template typename enable::type floor(T arg) { +//return functions::floor(arg); } +inline half floor(half arg) { return functions::floor(arg); } +inline half floor(expr arg) { return functions::floor(arg); } + +/// Nearest integer not greater in magnitude than half value. +/// \param arg half to round +/// \return nearest integer not greater in magnitude than \a arg +// template typename enable::type trunc(T arg) { +//return functions::trunc(arg); } +inline half trunc(half arg) { return functions::trunc(arg); } +inline half trunc(expr arg) { return functions::trunc(arg); } + +/// Nearest integer. +/// \param arg half to round +/// \return nearest integer, rounded away from zero in half-way cases +// template typename enable::type round(T arg) { +//return functions::round(arg); } +inline half round(half arg) { return functions::round(arg); } +inline half round(expr arg) { return functions::round(arg); } + +/// Nearest integer. +/// \param arg half to round +/// \return nearest integer, rounded away from zero in half-way cases +// template typename enable::type lround(T arg) { +//return functions::lround(arg); } +inline long lround(half arg) { return functions::lround(arg); } +inline long lround(expr arg) { return functions::lround(arg); } + +/// Nearest integer using half's internal rounding mode. +/// \param arg half expression to round +/// \return nearest integer using default rounding mode +// template typename enable::type nearbyint(T +//arg) { return functions::nearbyint(arg); } +inline half nearbyint(half arg) { return functions::rint(arg); } +inline half nearbyint(expr arg) { return functions::rint(arg); } + +/// Nearest integer using half's internal rounding mode. +/// \param arg half expression to round +/// \return nearest integer using default rounding mode +// template typename enable::type rint(T arg) { +//return functions::rint(arg); } +inline half rint(half arg) { return functions::rint(arg); } +inline half rint(expr arg) { return functions::rint(arg); } + +/// Nearest integer using half's internal rounding mode. +/// \param arg half expression to round +/// \return nearest integer using default rounding mode +// template typename enable::type lrint(T arg) { +//return functions::lrint(arg); } +inline long lrint(half arg) { return functions::lrint(arg); } +inline long lrint(expr arg) { return functions::lrint(arg); } +#if HALF_ENABLE_CPP11_LONG_LONG +/// Nearest integer. +/// \param arg half to round +/// \return nearest integer, rounded away from zero in half-way cases +// template typename enable::type llround(T +//arg) { return functions::llround(arg); } +inline long long llround(half arg) { return functions::llround(arg); } +inline long long llround(expr arg) { return functions::llround(arg); } + +/// Nearest integer using half's internal rounding mode. +/// \param arg half expression to round +/// \return nearest integer using default rounding mode +// template typename enable::type llrint(T +//arg) { return functions::llrint(arg); } +inline long long llrint(half arg) { return functions::llrint(arg); } +inline long long llrint(expr arg) { return functions::llrint(arg); } +#endif - /// Smallest finite value. - static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0xFBFF); } +/// \} +/// \name Floating point manipulation +/// \{ + +/// Decompress floating point number. +/// \param arg number to decompress +/// \param exp address to store exponent at +/// \return significant in range [0.5, 1) +// template typename enable::type frexp(T arg, +//int *exp) { return functions::frexp(arg, exp); } +inline half frexp(half arg, int *exp) { return functions::frexp(arg, exp); } +inline half frexp(expr arg, int *exp) { return functions::frexp(arg, exp); } + +/// Multiply by power of two. +/// \param arg number to modify +/// \param exp power of two to multiply with +/// \return \a arg multplied by 2 raised to \a exp +// template typename enable::type ldexp(T arg, +//int exp) { return functions::scalbln(arg, exp); } +inline half ldexp(half arg, int exp) { return functions::scalbln(arg, exp); } +inline half ldexp(expr arg, int exp) { return functions::scalbln(arg, exp); } + +/// Extract integer and fractional parts. +/// \param arg number to decompress +/// \param iptr address to store integer part at +/// \return fractional part +// template typename enable::type modf(T arg, +//half *iptr) { return functions::modf(arg, iptr); } +inline half modf(half arg, half *iptr) { return functions::modf(arg, iptr); } +inline half modf(expr arg, half *iptr) { return functions::modf(arg, iptr); } + +/// Multiply by power of two. +/// \param arg number to modify +/// \param exp power of two to multiply with +/// \return \a arg multplied by 2 raised to \a exp +// template typename enable::type scalbn(T arg, +//int exp) { return functions::scalbln(arg, exp); } +inline half scalbn(half arg, int exp) { return functions::scalbln(arg, exp); } +inline half scalbn(expr arg, int exp) { return functions::scalbln(arg, exp); } + +/// Multiply by power of two. +/// \param arg number to modify +/// \param exp power of two to multiply with +/// \return \a arg multplied by 2 raised to \a exp +// template typename enable::type scalbln(T arg, +//long exp) { return functions::scalbln(arg, exp); } +inline half scalbln(half arg, long exp) { return functions::scalbln(arg, exp); } +inline half scalbln(expr arg, long exp) { return functions::scalbln(arg, exp); } + +/// Extract exponent. +/// \param arg number to query +/// \return floating point exponent +/// \retval FP_ILOGB0 for zero +/// \retval FP_ILOGBNAN for NaN +/// \retval MAX_INT for infinity +// template typename enable::type ilogb(T arg) { +//return functions::ilogb(arg); } +inline int ilogb(half arg) { return functions::ilogb(arg); } +inline int ilogb(expr arg) { return functions::ilogb(arg); } + +/// Extract exponent. +/// \param arg number to query +/// \return floating point exponent +// template typename enable::type logb(T arg) { +//return functions::logb(arg); } +inline half logb(half arg) { return functions::logb(arg); } +inline half logb(expr arg) { return functions::logb(arg); } + +/// Next representable value. +/// \param from value to compute next representable value for +/// \param to direction towards which to compute next value +/// \return next representable value after \a from in direction towards \a to +// template typename enable::type +//nextafter(T from, U to) { return functions::nextafter(from, to); } +inline half nextafter(half from, half to) { + return functions::nextafter(from, to); +} +inline half nextafter(half from, expr to) { + return functions::nextafter(from, to); +} +inline half nextafter(expr from, half to) { + return functions::nextafter(from, to); +} +inline half nextafter(expr from, expr to) { + return functions::nextafter(from, to); +} - /// Largest finite value. - static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7BFF); } +/// Next representable value. +/// \param from value to compute next representable value for +/// \param to direction towards which to compute next value +/// \return next representable value after \a from in direction towards \a to +// template typename enable::type nexttoward(T +//from, long double to) { return functions::nexttoward(from, to); } +inline half nexttoward(half from, long double to) { + return functions::nexttoward(from, to); +} +inline half nexttoward(expr from, long double to) { + return functions::nexttoward(from, to); +} - /// Difference between one and next representable value. - static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x1400); } +/// Take sign. +/// \param x value to change sign for +/// \param y value to take sign from +/// \return value equal to \a x in magnitude and to \a y in sign +// template typename enable::type +//copysign(T x, U y) { return functions::copysign(x, y); } +inline half copysign(half x, half y) { return functions::copysign(x, y); } +inline half copysign(half x, expr y) { return functions::copysign(x, y); } +inline half copysign(expr x, half y) { return functions::copysign(x, y); } +inline half copysign(expr x, expr y) { return functions::copysign(x, y); } + +/// \} +/// \name Floating point classification +/// \{ + +/// Classify floating point value. +/// \param arg number to classify +/// \retval FP_ZERO for positive and negative zero +/// \retval FP_SUBNORMAL for subnormal numbers +/// \retval FP_INFINITY for positive and negative infinity +/// \retval FP_NAN for NaNs +/// \retval FP_NORMAL for all other (normal) values +// template typename enable::type fpclassify(T +//arg) { return functions::fpclassify(arg); } +inline int fpclassify(half arg) { return functions::fpclassify(arg); } +inline int fpclassify(expr arg) { return functions::fpclassify(arg); } + +/// Check if finite number. +/// \param arg number to check +/// \retval true if neither infinity nor NaN +/// \retval false else +// template typename enable::type isfinite(T arg) +//{ return functions::isfinite(arg); } +inline bool isfinite(half arg) { return functions::isfinite(arg); } +inline bool isfinite(expr arg) { return functions::isfinite(arg); } + +/// Check for infinity. +/// \param arg number to check +/// \retval true for positive or negative infinity +/// \retval false else +// template typename enable::type isinf(T arg) { +//return functions::isinf(arg); } +inline bool isinf(half arg) { return functions::isinf(arg); } +inline bool isinf(expr arg) { return functions::isinf(arg); } + +/// Check for NaN. +/// \param arg number to check +/// \retval true for NaNs +/// \retval false else +// template typename enable::type isnan(T arg) { +//return functions::isnan(arg); } +inline bool isnan(half arg) { return functions::isnan(arg); } +inline bool isnan(expr arg) { return functions::isnan(arg); } + +/// Check if normal number. +/// \param arg number to check +/// \retval true if normal number +/// \retval false if either subnormal, zero, infinity or NaN +// template typename enable::type isnormal(T arg) +//{ return functions::isnormal(arg); } +inline bool isnormal(half arg) { return functions::isnormal(arg); } +inline bool isnormal(expr arg) { return functions::isnormal(arg); } + +/// Check sign. +/// \param arg number to check +/// \retval true for negative number +/// \retval false for positive number +// template typename enable::type signbit(T arg) +//{ return functions::signbit(arg); } +inline bool signbit(half arg) { return functions::signbit(arg); } +inline bool signbit(expr arg) { return functions::signbit(arg); } + +/// \} +/// \name Comparison +/// \{ + +/// Comparison for greater than. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x greater than \a y +/// \retval false else +// template typename enable::type +//isgreater(T x, U y) { return functions::isgreater(x, y); } +inline bool isgreater(half x, half y) { return functions::isgreater(x, y); } +inline bool isgreater(half x, expr y) { return functions::isgreater(x, y); } +inline bool isgreater(expr x, half y) { return functions::isgreater(x, y); } +inline bool isgreater(expr x, expr y) { return functions::isgreater(x, y); } + +/// Comparison for greater equal. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x greater equal \a y +/// \retval false else +// template typename enable::type +//isgreaterequal(T x, U y) { return functions::isgreaterequal(x, y); } +inline bool isgreaterequal(half x, half y) { + return functions::isgreaterequal(x, y); +} +inline bool isgreaterequal(half x, expr y) { + return functions::isgreaterequal(x, y); +} +inline bool isgreaterequal(expr x, half y) { + return functions::isgreaterequal(x, y); +} +inline bool isgreaterequal(expr x, expr y) { + return functions::isgreaterequal(x, y); +} - /// Maximum rounding error. - static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW - { return half_float::half(half_float::detail::binary, (round_style==std::round_to_nearest) ? 0x3800 : 0x3C00); } +/// Comparison for less than. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x less than \a y +/// \retval false else +// template typename enable::type +//isless(T x, U y) { return functions::isless(x, y); } +inline bool isless(half x, half y) { return functions::isless(x, y); } +inline bool isless(half x, expr y) { return functions::isless(x, y); } +inline bool isless(expr x, half y) { return functions::isless(x, y); } +inline bool isless(expr x, expr y) { return functions::isless(x, y); } + +/// Comparison for less equal. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x less equal \a y +/// \retval false else +// template typename enable::type +//islessequal(T x, U y) { return functions::islessequal(x, y); } +inline bool islessequal(half x, half y) { return functions::islessequal(x, y); } +inline bool islessequal(half x, expr y) { return functions::islessequal(x, y); } +inline bool islessequal(expr x, half y) { return functions::islessequal(x, y); } +inline bool islessequal(expr x, expr y) { return functions::islessequal(x, y); } + +/// Comarison for less or greater. +/// \param x first operand +/// \param y second operand +/// \retval true if either less or greater +/// \retval false else +// template typename enable::type +//islessgreater(T x, U y) { return functions::islessgreater(x, y); } +inline bool islessgreater(half x, half y) { + return functions::islessgreater(x, y); +} +inline bool islessgreater(half x, expr y) { + return functions::islessgreater(x, y); +} +inline bool islessgreater(expr x, half y) { + return functions::islessgreater(x, y); +} +inline bool islessgreater(expr x, expr y) { + return functions::islessgreater(x, y); +} - /// Positive infinity. - static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7C00); } +/// Check if unordered. +/// \param x first operand +/// \param y second operand +/// \retval true if unordered (one or two NaN operands) +/// \retval false else +// template typename enable::type +//isunordered(T x, U y) { return functions::isunordered(x, y); } +inline bool isunordered(half x, half y) { return functions::isunordered(x, y); } +inline bool isunordered(half x, expr y) { return functions::isunordered(x, y); } +inline bool isunordered(expr x, half y) { return functions::isunordered(x, y); } +inline bool isunordered(expr x, expr y) { return functions::isunordered(x, y); } + +/// \name Casting +/// \{ + +/// Cast to or from half-precision floating point number. +/// This casts between [half](\ref half_float::half) and any built-in arithmetic +/// type. The values are converted +/// directly using the given rounding mode, without any roundtrip over `float` +/// that a `static_cast` would otherwise do. +/// It uses the default rounding mode. +/// +/// Using this cast with neither of the two types being a [half](\ref +/// half_float::half) or with any of the two types +/// not being a built-in arithmetic type (apart from [half](\ref +/// half_float::half), of course) results in a compiler +/// error and casting between [half](\ref half_float::half)s is just a no-op. +/// \tparam T destination type (half or built-in arithmetic type) +/// \tparam U source type (half or built-in arithmetic type) +/// \param arg value to cast +/// \return \a arg converted to destination type +template +T half_cast(U arg) { + return half_caster::cast(arg); +} - /// Quiet NaN. - static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7FFF); } +/// Cast to or from half-precision floating point number. +/// This casts between [half](\ref half_float::half) and any built-in arithmetic +/// type. The values are converted +/// directly using the given rounding mode, without any roundtrip over `float` +/// that a `static_cast` would otherwise do. +/// +/// Using this cast with neither of the two types being a [half](\ref +/// half_float::half) or with any of the two types +/// not being a built-in arithmetic type (apart from [half](\ref +/// half_float::half), of course) results in a compiler +/// error and casting between [half](\ref half_float::half)s is just a no-op. +/// \tparam T destination type (half or built-in arithmetic type) +/// \tparam R rounding mode to use. +/// \tparam U source type (half or built-in arithmetic type) +/// \param arg value to cast +/// \return \a arg converted to destination type +template +T half_cast(U arg) { + return half_caster::cast(arg); +} +/// \} +} - /// Signalling NaN. - static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7DFF); } +using detail::operator==; +using detail::operator!=; +using detail::operator<; +using detail::operator>; +using detail::operator<=; +using detail::operator>=; +using detail::operator+; +using detail::operator-; +using detail::operator*; +using detail::operator/; +using detail::operator<<; +using detail::operator>>; + +using detail::abs; +using detail::fabs; +using detail::fmod; +using detail::remainder; +using detail::remquo; +using detail::fma; +using detail::fmax; +using detail::fmin; +using detail::fdim; +using detail::nanh; +using detail::exp; +using detail::expm1; +using detail::exp2; +using detail::log; +using detail::log10; +using detail::log1p; +using detail::log2; +using detail::sqrt; +using detail::cbrt; +using detail::hypot; +using detail::pow; +using detail::sin; +using detail::cos; +using detail::tan; +using detail::asin; +using detail::acos; +using detail::atan; +using detail::atan2; +using detail::sinh; +using detail::cosh; +using detail::tanh; +using detail::asinh; +using detail::acosh; +using detail::atanh; +using detail::erf; +using detail::erfc; +using detail::lgamma; +using detail::tgamma; +using detail::ceil; +using detail::floor; +using detail::trunc; +using detail::round; +using detail::lround; +using detail::nearbyint; +using detail::rint; +using detail::lrint; +#if HALF_ENABLE_CPP11_LONG_LONG +using detail::llround; +using detail::llrint; +#endif +using detail::frexp; +using detail::ldexp; +using detail::modf; +using detail::scalbn; +using detail::scalbln; +using detail::ilogb; +using detail::logb; +using detail::nextafter; +using detail::nexttoward; +using detail::copysign; +using detail::fpclassify; +using detail::isfinite; +using detail::isinf; +using detail::isnan; +using detail::isnormal; +using detail::signbit; +using detail::isgreater; +using detail::isgreaterequal; +using detail::isless; +using detail::islessequal; +using detail::islessgreater; +using detail::isunordered; + +using detail::half_cast; +} - /// Smallest positive subnormal value. - static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0001); } - }; +/// Extensions to the C++ standard library. +namespace std { +/// Numeric limits for half-precision floats. +/// Because of the underlying single-precision implementation of many +/// operations, it inherits some properties from +/// `std::numeric_limits`. +template <> +class numeric_limits : public numeric_limits { + public: + /// Supports signed values. + static HALF_CONSTEXPR_CONST bool is_signed = true; + + /// Is not exact. + static HALF_CONSTEXPR_CONST bool is_exact = false; + + /// Doesn't provide modulo arithmetic. + static HALF_CONSTEXPR_CONST bool is_modulo = false; + + /// IEEE conformant. + static HALF_CONSTEXPR_CONST bool is_iec559 = true; + + /// Supports infinity. + static HALF_CONSTEXPR_CONST bool has_infinity = true; + + /// Supports quiet NaNs. + static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true; + + /// Supports subnormal values. + static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present; + + /// Rounding mode. + /// Due to the mix of internal single-precision computations (using the + /// rounding mode of the underlying + /// single-precision implementation) with the rounding mode of the + /// single-to-half conversions, the actual rounding + /// mode might be `std::round_indeterminate` if the default half-precision + /// rounding mode doesn't match the + /// single-precision rounding mode. + static HALF_CONSTEXPR_CONST float_round_style round_style = + (std::numeric_limits::round_style == half_float::half::round_style) + ? half_float::half::round_style + : round_indeterminate; + + /// Significant digits. + static HALF_CONSTEXPR_CONST int digits = 11; + + /// Significant decimal digits. + static HALF_CONSTEXPR_CONST int digits10 = 3; + + /// Required decimal digits to represent all possible values. + static HALF_CONSTEXPR_CONST int max_digits10 = 5; + + /// Number base. + static HALF_CONSTEXPR_CONST int radix = 2; + + /// One more than smallest exponent. + static HALF_CONSTEXPR_CONST int min_exponent = -13; + + /// Smallest normalized representable power of 10. + static HALF_CONSTEXPR_CONST int min_exponent10 = -4; + + /// One more than largest exponent + static HALF_CONSTEXPR_CONST int max_exponent = 16; + + /// Largest finitely representable power of 10. + static HALF_CONSTEXPR_CONST int max_exponent10 = 4; + + /// Smallest positive normal value. + static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x0400); + } + + /// Smallest finite value. + static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0xFBFF); + } + + /// Largest finite value. + static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x7BFF); + } + + /// Difference between one and next representable value. + static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x1400); + } + + /// Maximum rounding error. + static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW { + return half_float::half( + half_float::detail::binary, + (round_style == std::round_to_nearest) ? 0x3800 : 0x3C00); + } + + /// Positive infinity. + static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x7C00); + } + + /// Quiet NaN. + static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x7FFF); + } + + /// Signalling NaN. + static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x7DFF); + } + + /// Smallest positive subnormal value. + static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x0001); + } +}; #if HALF_ENABLE_CPP11_HASH - /// Hash function for half-precision floats. - /// This is only defined if C++11 `std::hash` is supported and enabled. - template<> struct hash //: unary_function - { - /// Type of function argument. - typedef half_float::half argument_type; - - /// Function return type. - typedef size_t result_type; - - /// Compute hash function. - /// \param arg half to hash - /// \return hash value - result_type operator()(argument_type arg) const - { return hash()(static_cast(arg.data_)&-(arg.data_!=0x8000)); } - }; +/// Hash function for half-precision floats. +/// This is only defined if C++11 `std::hash` is supported and enabled. +template <> +struct hash //: unary_function +{ + /// Type of function argument. + typedef half_float::half argument_type; + + /// Function return type. + typedef size_t result_type; + + /// Compute hash function. + /// \param arg half to hash + /// \return hash value + result_type operator()(argument_type arg) const { + return hash()(static_cast(arg.data_) & + -(arg.data_ != 0x8000)); + } +}; #endif } - #undef HALF_CONSTEXPR #undef HALF_CONSTEXPR_CONST #undef HALF_NOEXCEPT #undef HALF_NOTHROW #ifdef HALF_POP_WARNINGS - #pragma warning(pop) - #undef HALF_POP_WARNINGS +#pragma warning(pop) +#undef HALF_POP_WARNINGS #endif #endif diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 014e588..861f3bf 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -8,14 +8,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -25,34 +27,34 @@ // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "instructions.h" #include "half.h" #include "half.hpp" -#include "instructions.h" -#include "ptx_ir.h" #include "opcodes.h" +#include "ptx_ir.h" #include "ptx_sim.h" -typedef void * yyscan_t; +typedef void *yyscan_t; class ptx_recognizer; -#include "ptx.tab.h" -#include +#include +#include #include +#include +#include +#include +#include #include -#include -#include "cuda-math.h" +#include +#include +#include #include "../abstract_hardware_model.h" -#include "ptx_loader.h" -#include "cuda_device_printf.h" #include "../gpgpu-sim/gpu-sim.h" #include "../gpgpu-sim/shader.h" -#include -#include -#include -#include -#include -#include -#include +#include "cuda-math.h" +#include "cuda_device_printf.h" +#include "ptx.tab.h" +#include "ptx_loader.h" -//Jin: include device runtime for CDP +// Jin: include device runtime for CDP #include "cuda_device_runtime.h" #include @@ -60,5629 +62,6425 @@ class ptx_recognizer; using half_float::half; - - const char *g_opcode_string[NUM_OPCODES] = { -#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR, -#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR, +#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) STR, +#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) STR, #include "opcodes.def" #undef OP_DEF #undef OP_W_DEF }; -//Using profiled information::check the TensorCoreMatrixArrangement.xls for details -unsigned thread_group_offset(int thread,unsigned wmma_type,unsigned wmma_layout,unsigned type,int stride){ - - unsigned offset; - unsigned load_a_row[8]={0,128,0,128,64,192,64,192}; - unsigned load_a_col[8]={0,8,0,8,4,12,4,12}; - unsigned load_b_row[8]={0,8,0,8,4,12,4,12}; - unsigned load_b_col[8]={0,128,0,128,64,192,64,192}; - unsigned load_c_float_row[8]={0,128,8,136,64,192,72,200}; - unsigned load_c_float_col[8]={0,8,128,136,4,12,132,140}; - unsigned load_c_half_row[8]={0,128,8,136,64,192,72,200}; - unsigned load_c_half_col[8]={0,8,128,136,4,12,132,140}; - unsigned thread_group = thread/4; - unsigned in_tg_index = thread%4; - - switch(wmma_type){ - case LOAD_A: - if(wmma_layout==ROW) - offset=load_a_row[thread_group]+16*in_tg_index; - else - offset=load_a_col[thread_group]+16*in_tg_index; - break; - - - case LOAD_B: - if(wmma_layout==ROW) - offset=load_b_row[thread_group]+16*in_tg_index; - else - offset=load_b_col[thread_group]+16*in_tg_index; - break; - - case LOAD_C: - case STORE_D: - if(type==F16_TYPE){ - if(wmma_layout==ROW) - offset=load_c_half_row[thread_group]+16*in_tg_index; - else - offset=load_c_half_col[thread_group]+in_tg_index; - } - else{ - if(wmma_layout==ROW) - offset=load_c_float_row[thread_group]; - else - offset=load_c_float_col[thread_group]; - - switch(in_tg_index){ - case 0: - break; - case 1: - if(wmma_layout==ROW) - offset+=16; - else - offset+=1; - break; - case 2: - if(wmma_layout==ROW) - offset+=2; - else - offset+=32; - break; - case 3: - if(wmma_layout==ROW) - offset+=18; - else - offset+=33; - break; - default: - abort(); - } - } - break; - - default: - abort(); - - } - offset = (offset/16)*stride+offset%16; - return offset; -} - -int acc_float_offset(int index,int wmma_layout,int stride){ - - int c_row_offset[]={0,1,32,33,4,5,36,37}; - int c_col_offset[]={0,16,2,18,64,80,66,82}; - int offset; - - - if(wmma_layout==ROW) - offset=c_row_offset[index]; - else if(wmma_layout==COL) - offset=c_col_offset[index]; - else{ - printf("wrong layout"); - abort(); - } - offset = (offset/16)*stride+offset%16; - return offset; -} - -void inst_not_implemented( const ptx_instruction * pI ) ; -ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread); - -void sign_extend( ptx_reg_t &data, unsigned src_size, const operand_info &dst ); - -void ptx_thread_info::set_reg( const symbol *reg, const ptx_reg_t &value ) -{ - assert( reg != NULL ); - if( reg->name() == "_" ) return; - assert( !m_regs.empty() ); - assert( reg->uid() > 0 ); - m_regs.back()[ reg ] = value; - if (m_enable_debug_trace ) - m_debug_trace_regs_modified.back()[ reg ] = value; - m_last_set_operand_value = value; -} - -void ptx_thread_info::print_reg_thread(char * fname) -{ - - FILE *fp= fopen(fname,"w"); - assert(fp!=NULL); +// Using profiled information::check the TensorCoreMatrixArrangement.xls for +// details +unsigned thread_group_offset(int thread, unsigned wmma_type, + unsigned wmma_layout, unsigned type, int stride) { + unsigned offset; + unsigned load_a_row[8] = {0, 128, 0, 128, 64, 192, 64, 192}; + unsigned load_a_col[8] = {0, 8, 0, 8, 4, 12, 4, 12}; + unsigned load_b_row[8] = {0, 8, 0, 8, 4, 12, 4, 12}; + unsigned load_b_col[8] = {0, 128, 0, 128, 64, 192, 64, 192}; + unsigned load_c_float_row[8] = {0, 128, 8, 136, 64, 192, 72, 200}; + unsigned load_c_float_col[8] = {0, 8, 128, 136, 4, 12, 132, 140}; + unsigned load_c_half_row[8] = {0, 128, 8, 136, 64, 192, 72, 200}; + unsigned load_c_half_col[8] = {0, 8, 128, 136, 4, 12, 132, 140}; + unsigned thread_group = thread / 4; + unsigned in_tg_index = thread % 4; + + switch (wmma_type) { + case LOAD_A: + if (wmma_layout == ROW) + offset = load_a_row[thread_group] + 16 * in_tg_index; + else + offset = load_a_col[thread_group] + 16 * in_tg_index; + break; - int size = m_regs.size(); - - if(size>0) - { - reg_map_t reg = m_regs.back(); - - reg_map_t::const_iterator it; - for (it = reg.begin(); it != reg.end(); ++it) - { - const std::string &name = it->first->name(); - const std::string &dec= it->first->decl_location(); - unsigned size = it->first->get_size_in_bytes(); - fprintf(fp,"%s %llu %s %d\n", name.c_str(), it->second, dec.c_str(), size); - + case LOAD_B: + if (wmma_layout == ROW) + offset = load_b_row[thread_group] + 16 * in_tg_index; + else + offset = load_b_col[thread_group] + 16 * in_tg_index; + break; + + case LOAD_C: + case STORE_D: + if (type == F16_TYPE) { + if (wmma_layout == ROW) + offset = load_c_half_row[thread_group] + 16 * in_tg_index; + else + offset = load_c_half_col[thread_group] + in_tg_index; + } else { + if (wmma_layout == ROW) + offset = load_c_float_row[thread_group]; + else + offset = load_c_float_col[thread_group]; + + switch (in_tg_index) { + case 0: + break; + case 1: + if (wmma_layout == ROW) + offset += 16; + else + offset += 1; + break; + case 2: + if (wmma_layout == ROW) + offset += 2; + else + offset += 32; + break; + case 3: + if (wmma_layout == ROW) + offset += 18; + else + offset += 33; + break; + default: + abort(); } - //m_regs.pop_back(); + } + break; + + default: + abort(); + } + offset = (offset / 16) * stride + offset % 16; + return offset; +} + +int acc_float_offset(int index, int wmma_layout, int stride) { + int c_row_offset[] = {0, 1, 32, 33, 4, 5, 36, 37}; + int c_col_offset[] = {0, 16, 2, 18, 64, 80, 66, 82}; + int offset; + + if (wmma_layout == ROW) + offset = c_row_offset[index]; + else if (wmma_layout == COL) + offset = c_col_offset[index]; + else { + printf("wrong layout"); + abort(); + } + offset = (offset / 16) * stride + offset % 16; + return offset; +} + +void inst_not_implemented(const ptx_instruction *pI); +ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, + operand_info dstInfo, unsigned type, + ptx_thread_info *thread); + +void sign_extend(ptx_reg_t &data, unsigned src_size, const operand_info &dst); + +void ptx_thread_info::set_reg(const symbol *reg, const ptx_reg_t &value) { + assert(reg != NULL); + if (reg->name() == "_") return; + assert(!m_regs.empty()); + assert(reg->uid() > 0); + m_regs.back()[reg] = value; + if (m_enable_debug_trace) m_debug_trace_regs_modified.back()[reg] = value; + m_last_set_operand_value = value; +} + +void ptx_thread_info::print_reg_thread(char *fname) { + FILE *fp = fopen(fname, "w"); + assert(fp != NULL); + + int size = m_regs.size(); + + if (size > 0) { + reg_map_t reg = m_regs.back(); + + reg_map_t::const_iterator it; + for (it = reg.begin(); it != reg.end(); ++it) { + const std::string &name = it->first->name(); + const std::string &dec = it->first->decl_location(); + unsigned size = it->first->get_size_in_bytes(); + fprintf(fp, "%s %llu %s %d\n", name.c_str(), it->second, dec.c_str(), + size); + } + // m_regs.pop_back(); } fclose(fp); +} +void ptx_thread_info::resume_reg_thread(char *fname, symbol_table *symtab) { + FILE *fp2 = fopen(fname, "r"); + assert(fp2 != NULL); + // m_regs.push_back( reg_map_t() ); + char line[200]; + while (fgets(line, sizeof line, fp2) != NULL) { + symbol *reg; + char *pch; + pch = strtok(line, " "); + char *name = pch; + reg = symtab->lookup(name); + ptx_reg_t data; + pch = strtok(NULL, " "); + data = atoi(pch); + pch = strtok(NULL, " "); + pch = strtok(NULL, " "); + m_regs.back()[reg] = data; } + fclose(fp2); +} -void ptx_thread_info::resume_reg_thread(char * fname, symbol_table * symtab) -{ - - - FILE * fp2 = fopen(fname, "r"); - assert(fp2!=NULL); - //m_regs.push_back( reg_map_t() ); - char line [ 200 ]; - while ( fgets ( line, sizeof line, fp2 ) != NULL ) - { - symbol *reg; - char * pch; - pch = strtok (line," "); - char * name =pch; - reg= symtab->lookup(name); - ptx_reg_t data; - pch = strtok (NULL," "); - data = atoi(pch); - pch = strtok (NULL," "); - pch = strtok (NULL," "); - m_regs.back()[reg] = data; - } - fclose ( fp2 ); -} - - -ptx_reg_t ptx_thread_info::get_reg( const symbol *reg ) -{ - static bool unfound_register_warned = false; - assert( reg != NULL ); - assert( !m_regs.empty() ); - reg_map_t::iterator regs_iter = m_regs.back().find(reg); - if (regs_iter == m_regs.back().end()) { - assert( reg->type()->get_key().is_reg() ); - const std::string &name = reg->name(); - unsigned call_uid = m_callstack.back().m_call_uid; - ptx_reg_t uninit_reg; - uninit_reg.u32 = 0x0; - set_reg(reg, uninit_reg); // give it a value since we are going to warn the user anyway - std::string file_loc = get_location(); - if( !unfound_register_warned ) { - printf("GPGPU-Sim PTX: WARNING (%s) ** reading undefined register \'%s\' (cuid:%u). Setting to 0X00000000. This is okay if you are simulating the native ISA" - "\n", - file_loc.c_str(), name.c_str(), call_uid ); - unfound_register_warned = true; - } - regs_iter = m_regs.back().find(reg); - } - if (m_enable_debug_trace ) - m_debug_trace_regs_read.back()[ reg ] = regs_iter->second; - return regs_iter->second; -} - -ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_info dstInfo, unsigned opType, ptx_thread_info *thread, int derefFlag ) -{ - ptx_reg_t result, tmp; - - - if(op.get_double_operand_type() == 0) { - if(((opType != BB128_TYPE) && (opType != BB64_TYPE) && (opType != FF64_TYPE)) || (op.get_addr_space() != undefined_space)) { - if ( op.is_reg() ) { - result = get_reg( op.get_symbol() ); - } else if ( op.is_builtin()) { - result.u32 = get_builtin( op.get_int(), op.get_addr_offset() ); - } else if(op.is_immediate_address()){ - result.u64 = op.get_addr_offset(); - } else if ( op.is_memory_operand() ) { - // a few options here... - const symbol *sym = op.get_symbol(); - const type_info *type = sym->type(); - const type_info_key &info = type->get_key(); - - if ( info.is_reg() ) { - const symbol *name = op.get_symbol(); - result.u64 = get_reg(name).u64 + op.get_addr_offset(); - } else if ( info.is_param_kernel() ) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if ( info.is_param_local() ) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if ( info.is_global() ) { - assert( op.get_addr_offset() == 0 ); - result.u64 = sym->get_address(); - } else if ( info.is_local() ) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if ( info.is_const() ) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if ( op.is_shared() ) { - result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); - } else if ( op.is_sstarr() ) { - result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); - } else { - const char *name = op.name().c_str(); - printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory operand type for %s\n", name ); - abort(); - } +ptx_reg_t ptx_thread_info::get_reg(const symbol *reg) { + static bool unfound_register_warned = false; + assert(reg != NULL); + assert(!m_regs.empty()); + reg_map_t::iterator regs_iter = m_regs.back().find(reg); + if (regs_iter == m_regs.back().end()) { + assert(reg->type()->get_key().is_reg()); + const std::string &name = reg->name(); + unsigned call_uid = m_callstack.back().m_call_uid; + ptx_reg_t uninit_reg; + uninit_reg.u32 = 0x0; + set_reg(reg, uninit_reg); // give it a value since we are going to warn the + // user anyway + std::string file_loc = get_location(); + if (!unfound_register_warned) { + printf( + "GPGPU-Sim PTX: WARNING (%s) ** reading undefined register \'%s\' " + "(cuid:%u). Setting to 0X00000000. This is okay if you are " + "simulating the native ISA" + "\n", + file_loc.c_str(), name.c_str(), call_uid); + unfound_register_warned = true; + } + regs_iter = m_regs.back().find(reg); + } + if (m_enable_debug_trace) + m_debug_trace_regs_read.back()[reg] = regs_iter->second; + return regs_iter->second; +} - } else if ( op.is_literal() ) { - result = op.get_literal_value(); - } else if ( op.is_label() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_shared() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_sstarr() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_const() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_global() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_local() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_function_address() ) { - result.u64 = (size_t)op.get_symbol()->get_pc(); - } else if ( op.is_param_kernel()) { - result.u64 = op.get_symbol()->get_address(); - }else { - const char *name = op.name().c_str(); - const symbol *sym2 = op.get_symbol(); - const type_info *type2 = sym2->type(); - const type_info_key &info2 = type2->get_key(); - if ( info2.is_param_kernel() ) { - result.u64 = sym2->get_address()+ op.get_addr_offset(); - } - else{ - printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand type for %s\n", name ); - assert(0); - } - } - - if(op.get_operand_lohi() == 1) - result.u64 = result.u64 & 0xFFFF; - else if(op.get_operand_lohi() == 2) - result.u64 = (result.u64>>16) & 0xFFFF; - } else if (opType == BB128_TYPE) { - // b128 - result.u128.lowest = get_reg( op.vec_symbol(0) ).u32; - result.u128.low = get_reg( op.vec_symbol(1) ).u32; - result.u128.high = get_reg( op.vec_symbol(2) ).u32; - result.u128.highest = get_reg( op.vec_symbol(3) ).u32; +ptx_reg_t ptx_thread_info::get_operand_value(const operand_info &op, + operand_info dstInfo, + unsigned opType, + ptx_thread_info *thread, + int derefFlag) { + ptx_reg_t result, tmp; + + if (op.get_double_operand_type() == 0) { + if (((opType != BB128_TYPE) && (opType != BB64_TYPE) && + (opType != FF64_TYPE)) || + (op.get_addr_space() != undefined_space)) { + if (op.is_reg()) { + result = get_reg(op.get_symbol()); + } else if (op.is_builtin()) { + result.u32 = get_builtin(op.get_int(), op.get_addr_offset()); + } else if (op.is_immediate_address()) { + result.u64 = op.get_addr_offset(); + } else if (op.is_memory_operand()) { + // a few options here... + const symbol *sym = op.get_symbol(); + const type_info *type = sym->type(); + const type_info_key &info = type->get_key(); + + if (info.is_reg()) { + const symbol *name = op.get_symbol(); + result.u64 = get_reg(name).u64 + op.get_addr_offset(); + } else if (info.is_param_kernel()) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if (info.is_param_local()) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if (info.is_global()) { + assert(op.get_addr_offset() == 0); + result.u64 = sym->get_address(); + } else if (info.is_local()) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if (info.is_const()) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if (op.is_shared()) { + result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); + } else if (op.is_sstarr()) { + result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); + } else { + const char *name = op.name().c_str(); + printf( + "GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory " + "operand type for %s\n", + name); + abort(); + } + + } else if (op.is_literal()) { + result = op.get_literal_value(); + } else if (op.is_label()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_shared()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_sstarr()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_const()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_global()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_local()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_function_address()) { + result.u64 = (size_t)op.get_symbol()->get_pc(); + } else if (op.is_param_kernel()) { + result.u64 = op.get_symbol()->get_address(); } else { - // bb64 or ff64 - result.bits.ls = get_reg( op.vec_symbol(0) ).u32; - result.bits.ms = get_reg( op.vec_symbol(1) ).u32; + const char *name = op.name().c_str(); + const symbol *sym2 = op.get_symbol(); + const type_info *type2 = sym2->type(); + const type_info_key &info2 = type2->get_key(); + if (info2.is_param_kernel()) { + result.u64 = sym2->get_address() + op.get_addr_offset(); + } else { + printf( + "GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand " + "type for %s\n", + name); + assert(0); + } } - } else if (op.get_double_operand_type() == 1) { - ptx_reg_t firstHalf, secondHalf; - firstHalf.u64 = get_reg( op.vec_symbol(0) ).u64; - secondHalf.u64 = get_reg( op.vec_symbol(1) ).u64; - if(op.get_operand_lohi() == 1) - secondHalf.u64 = secondHalf.u64 & 0xFFFF; - else if(op.get_operand_lohi() == 2) - secondHalf.u64 = (secondHalf.u64>>16) & 0xFFFF; - result.u64 = firstHalf.u64 + secondHalf.u64; - } else if (op.get_double_operand_type() == 2) { - // s[reg1 += reg2] - // reg1 is incremented after value is returned: the value returned is s[reg1] - ptx_reg_t firstHalf, secondHalf; - firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; - secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; - if(op.get_operand_lohi() == 1) - secondHalf.u64 = secondHalf.u64 & 0xFFFF; - else if(op.get_operand_lohi() == 2) - secondHalf.u64 = (secondHalf.u64>>16) & 0xFFFF; - result.u64 = firstHalf.u64; - firstHalf.u64 = firstHalf.u64 + secondHalf.u64; - set_reg(op.vec_symbol(0),firstHalf); - } else if (op.get_double_operand_type() == 3) { - // s[reg += immediate] - // reg is incremented after value is returned: the value returned is s[reg] - ptx_reg_t firstHalf; - firstHalf.u64 = get_reg(op.get_symbol()).u64; - result.u64 = firstHalf.u64; - firstHalf.u64 = firstHalf.u64 + op.get_addr_offset(); - set_reg(op.get_symbol(),firstHalf); - } - - ptx_reg_t finalResult; - memory_space *mem = NULL; - size_t size=0; - int t=0; - finalResult.u64=0; - - //complete other cases for reading from memory, such as reading from other const memory - if((op.get_addr_space() == global_space)&&(derefFlag)) { - // global memory - g[4], g[$r0] - mem = thread->get_global_memory(); - type_info_key::type_decode(opType,size,t); - mem->read(result.u32,size/8,&finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = global_space; - - if( opType == S16_TYPE || opType == S32_TYPE ) - sign_extend(finalResult,size,dstInfo); - } else if((op.get_addr_space() == shared_space)&&(derefFlag)) { - // shared memory - s[4], s[$r0] - mem = thread->m_shared_mem; - type_info_key::type_decode(opType,size,t); - mem->read(result.u32,size/8,&finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = shared_space; - - if( opType == S16_TYPE || opType == S32_TYPE ) - sign_extend(finalResult,size,dstInfo); - } else if((op.get_addr_space() == const_space)&&(derefFlag)) { - // const memory - ce0c1[4], ce0c1[$r0] - mem = thread->get_global_memory(); - type_info_key::type_decode(opType,size,t); - mem->read((result.u32 + op.get_const_mem_offset()),size/8,&finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = const_space; - if( opType == S16_TYPE || opType == S32_TYPE ) - sign_extend(finalResult,size,dstInfo); - } else if((op.get_addr_space() == local_space)&&(derefFlag)) { - // local memory - l0[4], l0[$r0] - mem = thread->m_local_mem; - type_info_key::type_decode(opType,size,t); - mem->read(result.u32,size/8,&finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = local_space; - if( opType == S16_TYPE || opType == S32_TYPE ) - sign_extend(finalResult,size,dstInfo); - } else { - finalResult = result; - } - - if((op.get_operand_neg() == true)&&(derefFlag)) { - switch( opType ) { + + if (op.get_operand_lohi() == 1) + result.u64 = result.u64 & 0xFFFF; + else if (op.get_operand_lohi() == 2) + result.u64 = (result.u64 >> 16) & 0xFFFF; + } else if (opType == BB128_TYPE) { + // b128 + result.u128.lowest = get_reg(op.vec_symbol(0)).u32; + result.u128.low = get_reg(op.vec_symbol(1)).u32; + result.u128.high = get_reg(op.vec_symbol(2)).u32; + result.u128.highest = get_reg(op.vec_symbol(3)).u32; + } else { + // bb64 or ff64 + result.bits.ls = get_reg(op.vec_symbol(0)).u32; + result.bits.ms = get_reg(op.vec_symbol(1)).u32; + } + } else if (op.get_double_operand_type() == 1) { + ptx_reg_t firstHalf, secondHalf; + firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; + secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; + if (op.get_operand_lohi() == 1) + secondHalf.u64 = secondHalf.u64 & 0xFFFF; + else if (op.get_operand_lohi() == 2) + secondHalf.u64 = (secondHalf.u64 >> 16) & 0xFFFF; + result.u64 = firstHalf.u64 + secondHalf.u64; + } else if (op.get_double_operand_type() == 2) { + // s[reg1 += reg2] + // reg1 is incremented after value is returned: the value returned is + // s[reg1] + ptx_reg_t firstHalf, secondHalf; + firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; + secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; + if (op.get_operand_lohi() == 1) + secondHalf.u64 = secondHalf.u64 & 0xFFFF; + else if (op.get_operand_lohi() == 2) + secondHalf.u64 = (secondHalf.u64 >> 16) & 0xFFFF; + result.u64 = firstHalf.u64; + firstHalf.u64 = firstHalf.u64 + secondHalf.u64; + set_reg(op.vec_symbol(0), firstHalf); + } else if (op.get_double_operand_type() == 3) { + // s[reg += immediate] + // reg is incremented after value is returned: the value returned is s[reg] + ptx_reg_t firstHalf; + firstHalf.u64 = get_reg(op.get_symbol()).u64; + result.u64 = firstHalf.u64; + firstHalf.u64 = firstHalf.u64 + op.get_addr_offset(); + set_reg(op.get_symbol(), firstHalf); + } + + ptx_reg_t finalResult; + memory_space *mem = NULL; + size_t size = 0; + int t = 0; + finalResult.u64 = 0; + + // complete other cases for reading from memory, such as reading from other + // const memory + if ((op.get_addr_space() == global_space) && (derefFlag)) { + // global memory - g[4], g[$r0] + mem = thread->get_global_memory(); + type_info_key::type_decode(opType, size, t); + mem->read(result.u32, size / 8, &finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = global_space; + + if (opType == S16_TYPE || opType == S32_TYPE) + sign_extend(finalResult, size, dstInfo); + } else if ((op.get_addr_space() == shared_space) && (derefFlag)) { + // shared memory - s[4], s[$r0] + mem = thread->m_shared_mem; + type_info_key::type_decode(opType, size, t); + mem->read(result.u32, size / 8, &finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = shared_space; + + if (opType == S16_TYPE || opType == S32_TYPE) + sign_extend(finalResult, size, dstInfo); + } else if ((op.get_addr_space() == const_space) && (derefFlag)) { + // const memory - ce0c1[4], ce0c1[$r0] + mem = thread->get_global_memory(); + type_info_key::type_decode(opType, size, t); + mem->read((result.u32 + op.get_const_mem_offset()), size / 8, + &finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = const_space; + if (opType == S16_TYPE || opType == S32_TYPE) + sign_extend(finalResult, size, dstInfo); + } else if ((op.get_addr_space() == local_space) && (derefFlag)) { + // local memory - l0[4], l0[$r0] + mem = thread->m_local_mem; + type_info_key::type_decode(opType, size, t); + mem->read(result.u32, size / 8, &finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = local_space; + if (opType == S16_TYPE || opType == S32_TYPE) + sign_extend(finalResult, size, dstInfo); + } else { + finalResult = result; + } + + if ((op.get_operand_neg() == true) && (derefFlag)) { + switch (opType) { // Default to f32 for now, need to add support for others case S8_TYPE: case U8_TYPE: case B8_TYPE: - finalResult.s8 = -finalResult.s8; - break; + finalResult.s8 = -finalResult.s8; + break; case S16_TYPE: case U16_TYPE: case B16_TYPE: - finalResult.s16 = -finalResult.s16; - break; + finalResult.s16 = -finalResult.s16; + break; case S32_TYPE: case U32_TYPE: case B32_TYPE: - finalResult.s32 = -finalResult.s32; - break; + finalResult.s32 = -finalResult.s32; + break; case S64_TYPE: case U64_TYPE: case B64_TYPE: - finalResult.s64 = -finalResult.s64; - break; + finalResult.s64 = -finalResult.s64; + break; case F16_TYPE: - finalResult.f16 = -finalResult.f16; - break; + finalResult.f16 = -finalResult.f16; + break; case F32_TYPE: - finalResult.f32 = -finalResult.f32; - break; + finalResult.f32 = -finalResult.f32; + break; case F64_TYPE: case FF64_TYPE: - finalResult.f64 = -finalResult.f64; - break; + finalResult.f64 = -finalResult.f64; + break; default: - assert(0); - } + assert(0); + } + } + + return finalResult; +} - } - - return finalResult; - -} - -unsigned get_operand_nbits( const operand_info &op ) -{ - if ( op.is_reg() ) { - const symbol *sym = op.get_symbol(); - const type_info *typ = sym->type(); - type_info_key t = typ->get_key(); - switch( t.scalar_type() ) { - case PRED_TYPE: - return 1; - case B8_TYPE: case S8_TYPE: case U8_TYPE: - return 8; - case S16_TYPE: case U16_TYPE: case F16_TYPE: case B16_TYPE: - return 16; - case S32_TYPE: case U32_TYPE: case F32_TYPE: case B32_TYPE: - return 32; - case S64_TYPE: case U64_TYPE: case F64_TYPE: case B64_TYPE: - return 64; +unsigned get_operand_nbits(const operand_info &op) { + if (op.is_reg()) { + const symbol *sym = op.get_symbol(); + const type_info *typ = sym->type(); + type_info_key t = typ->get_key(); + switch (t.scalar_type()) { + case PRED_TYPE: + return 1; + case B8_TYPE: + case S8_TYPE: + case U8_TYPE: + return 8; + case S16_TYPE: + case U16_TYPE: + case F16_TYPE: + case B16_TYPE: + return 16; + case S32_TYPE: + case U32_TYPE: + case F32_TYPE: + case B32_TYPE: + return 32; + case S64_TYPE: + case U64_TYPE: + case F64_TYPE: + case B64_TYPE: + return 64; default: - printf("ERROR: unknown register type\n"); - fflush(stdout); - abort(); - } - } else { - printf("ERROR: Need to implement get_operand_nbits() for currently unsupported operand_info type\n"); - fflush(stdout); - abort(); - } + printf("ERROR: unknown register type\n"); + fflush(stdout); + abort(); + } + } else { + printf( + "ERROR: Need to implement get_operand_nbits() for currently " + "unsupported operand_info type\n"); + fflush(stdout); + abort(); + } - return 0; + return 0; } -void ptx_thread_info::get_vector_operand_values( const operand_info &op, ptx_reg_t* ptx_regs, unsigned num_elements ) -{ - assert( op.is_vector() ); - assert( num_elements <= 8 ); +void ptx_thread_info::get_vector_operand_values(const operand_info &op, + ptx_reg_t *ptx_regs, + unsigned num_elements) { + assert(op.is_vector()); + assert(num_elements <= 8); + + for (int idx = num_elements - 1; idx >= 0; --idx) { + const symbol *sym = NULL; + sym = op.vec_symbol(idx); + if (strcmp(sym->name().c_str(), "_") != 0) { + reg_map_t::iterator reg_iter = m_regs.back().find(sym); + assert(reg_iter != m_regs.back().end()); + ptx_regs[idx] = reg_iter->second; + } + } +} - for (int idx = num_elements - 1; idx >= 0; --idx) { - const symbol *sym = NULL; - sym = op.vec_symbol(idx); - if( strcmp(sym->name().c_str(),"_") != 0) { - reg_map_t::iterator reg_iter = m_regs.back().find(sym); - assert( reg_iter != m_regs.back().end() ); - ptx_regs[idx] = reg_iter->second; - } - } +void sign_extend(ptx_reg_t &data, unsigned src_size, const operand_info &dst) { + if (!dst.is_reg()) return; + unsigned dst_size = get_operand_nbits(dst); + if (src_size >= dst_size) return; + // src_size < dst_size + unsigned long long mask = 1; + mask <<= (src_size - 1); + if ((mask & data.u64) == 0) { + // no need to sign extend + return; + } + // need to sign extend + mask = 1; + mask <<= dst_size - src_size; + mask -= 1; + mask <<= src_size; + data.u64 |= mask; } -void sign_extend( ptx_reg_t &data, unsigned src_size, const operand_info &dst ) -{ - if( !dst.is_reg() ) - return; - unsigned dst_size = get_operand_nbits( dst ); - if( src_size >= dst_size ) - return; - // src_size < dst_size - unsigned long long mask = 1; - mask <<= (src_size-1); - if( (mask & data.u64) == 0 ) { - // no need to sign extend - return; - } - // need to sign extend - mask = 1; - mask <<= dst_size-src_size; - mask -= 1; - mask <<= src_size; - data.u64 |= mask; +void ptx_thread_info::set_operand_value(const operand_info &dst, + const ptx_reg_t &data, unsigned type, + ptx_thread_info *thread, + const ptx_instruction *pI, int overflow, + int carry) { + thread->set_operand_value(dst, data, type, thread, pI); + + if (dst.get_double_operand_type() == -2) { + ptx_reg_t predValue; + + const symbol *sym = dst.vec_symbol(0); + predValue.u64 = (m_regs.back()[sym].u64) & ~(0x0C); + predValue.u64 |= ((overflow & 0x01) << 3); + predValue.u64 |= ((carry & 0x01) << 2); + + set_reg(sym, predValue); + } else if (dst.get_double_operand_type() == 0) { + // intentionally do nothing + } else { + printf("Unexpected double destination\n"); + assert(0); + } } -void ptx_thread_info::set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI, int overflow, int carry ) -{ - thread->set_operand_value( dst, data, type, thread, pI ); +void ptx_thread_info::set_operand_value(const operand_info &dst, + const ptx_reg_t &data, unsigned type, + ptx_thread_info *thread, + const ptx_instruction *pI) { + ptx_reg_t dstData; + memory_space *mem = NULL; + size_t size; + int t; + + type_info_key::type_decode(type, size, t); + + /*complete this section for other cases*/ + if (dst.get_addr_space() == undefined_space) { + ptx_reg_t setValue; + setValue.u64 = data.u64; + + // Double destination in set instruction ($p0|$p1) - second is negation of + // first + if (dst.get_double_operand_type() == -1) { + ptx_reg_t setValue2; + const symbol *name1 = dst.vec_symbol(0); + const symbol *name2 = dst.vec_symbol(1); + + if ((type == F16_TYPE) || (type == F32_TYPE) || (type == F64_TYPE) || + (type == FF64_TYPE)) { + setValue2.f32 = (setValue.u64 == 0) ? 1.0f : 0.0f; + } else { + setValue2.u32 = (setValue.u64 == 0) ? 0xFFFFFFFF : 0; + } - if (dst.get_double_operand_type() == -2) - { - ptx_reg_t predValue; - - const symbol *sym = dst.vec_symbol(0); - predValue.u64 = (m_regs.back()[ sym ].u64) & ~(0x0C); - predValue.u64 |= ((overflow & 0x01)<<3); - predValue.u64 |= ((carry & 0x01)<<2); - - set_reg(sym,predValue); - } - else if (dst.get_double_operand_type() == 0) - { - //intentionally do nothing + set_reg(name1, setValue); + set_reg(name2, setValue2); } - else - { - printf("Unexpected double destination\n"); - assert(0); + + // Double destination in cvt,shr,mul,etc. instruction ($p0|$r4) - second + // register operand receives data, first predicate operand + // is set as $p0=($r4!=0) + // Also for Double destination in set instruction ($p0/$r1) + else if ((dst.get_double_operand_type() == -2) || + (dst.get_double_operand_type() == -3)) { + ptx_reg_t predValue; + const symbol *predName = dst.vec_symbol(0); + const symbol *regName = dst.vec_symbol(1); + predValue.u64 = 0; + + switch (type) { + case S8_TYPE: + if ((setValue.s8 & 0x7F) == 0) predValue.u64 |= 1; + break; + case S16_TYPE: + if ((setValue.s16 & 0x7FFF) == 0) predValue.u64 |= 1; + break; + case S32_TYPE: + if ((setValue.s32 & 0x7FFFFFFF) == 0) predValue.u64 |= 1; + break; + case S64_TYPE: + if ((setValue.s64 & 0x7FFFFFFFFFFFFFFF) == 0) predValue.u64 |= 1; + break; + case U8_TYPE: + case B8_TYPE: + if (setValue.u8 == 0) predValue.u64 |= 1; + break; + case U16_TYPE: + case B16_TYPE: + if (setValue.u16 == 0) predValue.u64 |= 1; + break; + case U32_TYPE: + case B32_TYPE: + if (setValue.u32 == 0) predValue.u64 |= 1; + break; + case U64_TYPE: + case B64_TYPE: + if (setValue.u64 == 0) predValue.u64 |= 1; + break; + case F16_TYPE: + if (setValue.f16 == 0) predValue.u64 |= 1; + break; + case F32_TYPE: + if (setValue.f32 == 0) predValue.u64 |= 1; + break; + case F64_TYPE: + case FF64_TYPE: + if (setValue.f64 == 0) predValue.u64 |= 1; + break; + default: + assert(0); + break; + } + + if ((type == S8_TYPE) || (type == S16_TYPE) || (type == S32_TYPE) || + (type == S64_TYPE) || (type == U8_TYPE) || (type == U16_TYPE) || + (type == U32_TYPE) || (type == U64_TYPE) || (type == B8_TYPE) || + (type == B16_TYPE) || (type == B32_TYPE) || (type == B64_TYPE)) { + if ((setValue.u32 & (1 << (size - 1))) != 0) predValue.u64 |= 1 << 1; + } + if (type == F32_TYPE) { + if (setValue.f32 < 0) predValue.u64 |= 1 << 1; + } + + if (dst.get_operand_lohi() == 1) { + setValue.u64 = + ((m_regs.back()[regName].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); + } else if (dst.get_operand_lohi() == 2) { + setValue.u64 = ((m_regs.back()[regName].u64) & (~(0xFFFF0000))) + + ((data.u64 << 16) & 0xFFFF0000); + } + + set_reg(predName, predValue); + set_reg(regName, setValue); + } else if (type == BB128_TYPE) { + // b128 stuff here. + ptx_reg_t setValue2, setValue3, setValue4; + setValue.u64 = 0; + setValue2.u64 = 0; + setValue3.u64 = 0; + setValue4.u64 = 0; + setValue.u32 = data.u128.lowest; + setValue2.u32 = data.u128.low; + setValue3.u32 = data.u128.high; + setValue4.u32 = data.u128.highest; + + const symbol *name1, *name2, *name3, *name4 = NULL; + + name1 = dst.vec_symbol(0); + name2 = dst.vec_symbol(1); + name3 = dst.vec_symbol(2); + name4 = dst.vec_symbol(3); + + set_reg(name1, setValue); + set_reg(name2, setValue2); + set_reg(name3, setValue3); + set_reg(name4, setValue4); + } else if (type == BB64_TYPE || type == FF64_TYPE) { + // ptxplus version of storing 64 bit values to registers stores to two + // adjacent registers + ptx_reg_t setValue2; + setValue.u32 = 0; + setValue2.u32 = 0; + + setValue.u32 = data.bits.ls; + setValue2.u32 = data.bits.ms; + + const symbol *name1, *name2 = NULL; + + name1 = dst.vec_symbol(0); + name2 = dst.vec_symbol(1); + + set_reg(name1, setValue); + set_reg(name2, setValue2); + } else { + if (dst.get_operand_lohi() == 1) { + setValue.u64 = ((m_regs.back()[dst.get_symbol()].u64) & (~(0xFFFF))) + + (data.u64 & 0xFFFF); + } else if (dst.get_operand_lohi() == 2) { + setValue.u64 = + ((m_regs.back()[dst.get_symbol()].u64) & (~(0xFFFF0000))) + + ((data.u64 << 16) & 0xFFFF0000); + } + set_reg(dst.get_symbol(), setValue); } + } -} + // global memory - g[4], g[$r0] + else if (dst.get_addr_space() == global_space) { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->get_global_memory(); + type_info_key::type_decode(type, size, t); + + mem->write(dstData.u32, size / 8, &data.u128, thread, pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = global_space; + } -void ptx_thread_info::set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI ) -{ - ptx_reg_t dstData; - memory_space *mem = NULL; - size_t size; - int t; + // shared memory - s[4], s[$r0] + else if (dst.get_addr_space() == shared_space) { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->m_shared_mem; + type_info_key::type_decode(type, size, t); - type_info_key::type_decode(type,size,t); + mem->write(dstData.u32, size / 8, &data.u128, thread, pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = shared_space; + } - /*complete this section for other cases*/ - if(dst.get_addr_space() == undefined_space) - { - ptx_reg_t setValue; - setValue.u64 = data.u64; + // local memory - l0[4], l0[$r0] + else if (dst.get_addr_space() == local_space) { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->m_local_mem; + type_info_key::type_decode(type, size, t); - // Double destination in set instruction ($p0|$p1) - second is negation of first - if (dst.get_double_operand_type() == -1) - { - ptx_reg_t setValue2; - const symbol *name1 = dst.vec_symbol(0); - const symbol *name2 = dst.vec_symbol(1); + mem->write(dstData.u32, size / 8, &data.u128, thread, pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = local_space; + } - if ( (type==F16_TYPE)||(type==F32_TYPE)||(type==F64_TYPE)||(type==FF64_TYPE) ) { - setValue2.f32 = (setValue.u64==0)?1.0f:0.0f; - } else { - setValue2.u32 = (setValue.u64==0)?0xFFFFFFFF:0; - } + else { + printf("Destination stores to unknown location."); + assert(0); + } +} - set_reg(name1,setValue); - set_reg(name2,setValue2); +void ptx_thread_info::set_vector_operand_values(const operand_info &dst, + const ptx_reg_t &data1, + const ptx_reg_t &data2, + const ptx_reg_t &data3, + const ptx_reg_t &data4) { + unsigned num_elements = dst.get_vect_nelem(); + if (num_elements > 0) { + set_reg(dst.vec_symbol(0), data1); + if (num_elements > 1) { + set_reg(dst.vec_symbol(1), data2); + if (num_elements > 2) { + set_reg(dst.vec_symbol(2), data3); + if (num_elements > 3) { + set_reg(dst.vec_symbol(3), data4); + } } + } + } - // Double destination in cvt,shr,mul,etc. instruction ($p0|$r4) - second register operand receives data, first predicate operand - // is set as $p0=($r4!=0) - // Also for Double destination in set instruction ($p0/$r1) - else if ((dst.get_double_operand_type() == -2)||(dst.get_double_operand_type() == -3)) - { - ptx_reg_t predValue; - const symbol *predName = dst.vec_symbol(0); - const symbol *regName = dst.vec_symbol(1); - predValue.u64 = 0; - - switch ( type ) { - case S8_TYPE: - if((setValue.s8 & 0x7F) == 0) - predValue.u64 |= 1; - break; - case S16_TYPE: - if((setValue.s16 & 0x7FFF) == 0) - predValue.u64 |= 1; - break; - case S32_TYPE: - if((setValue.s32 & 0x7FFFFFFF) == 0) - predValue.u64 |= 1; - break; - case S64_TYPE: - if((setValue.s64 & 0x7FFFFFFFFFFFFFFF) == 0) - predValue.u64 |= 1; - break; - case U8_TYPE: - case B8_TYPE: - if(setValue.u8 == 0) - predValue.u64 |= 1; - break; - case U16_TYPE: - case B16_TYPE: - if(setValue.u16 == 0) - predValue.u64 |= 1; - break; - case U32_TYPE: - case B32_TYPE: - if(setValue.u32 == 0) - predValue.u64 |= 1; - break; - case U64_TYPE: - case B64_TYPE: - if(setValue.u64 == 0) - predValue.u64 |= 1; - break; - case F16_TYPE: - if(setValue.f16 == 0) - predValue.u64 |= 1; - break; - case F32_TYPE: - if(setValue.f32 == 0) - predValue.u64 |= 1; - break; - case F64_TYPE: - case FF64_TYPE: - if(setValue.f64 == 0) - predValue.u64 |= 1; - break; - default: assert(0); break; - } + m_last_set_operand_value = data1; +} +void ptx_thread_info::set_wmma_vector_operand_values( + const operand_info &dst, const ptx_reg_t &data1, const ptx_reg_t &data2, + const ptx_reg_t &data3, const ptx_reg_t &data4, const ptx_reg_t &data5, + const ptx_reg_t &data6, const ptx_reg_t &data7, const ptx_reg_t &data8) { + unsigned num_elements = dst.get_vect_nelem(); + if (num_elements == 8) { + set_reg(dst.vec_symbol(0), data1); + set_reg(dst.vec_symbol(1), data2); + set_reg(dst.vec_symbol(2), data3); + set_reg(dst.vec_symbol(3), data4); + set_reg(dst.vec_symbol(4), data5); + set_reg(dst.vec_symbol(5), data6); + set_reg(dst.vec_symbol(6), data7); + set_reg(dst.vec_symbol(7), data8); + } else { + printf("error:set_wmma_vector_operands"); + } + m_last_set_operand_value = data8; +} - if ( (type==S8_TYPE)||(type==S16_TYPE)||(type==S32_TYPE)||(type==S64_TYPE)|| - (type==U8_TYPE)||(type==U16_TYPE)||(type==U32_TYPE)||(type==U64_TYPE)|| - (type==B8_TYPE)||(type==B16_TYPE)||(type==B32_TYPE)||(type==B64_TYPE)) { - if((setValue.u32 & (1<<(size-1))) != 0) - predValue.u64 |= 1<<1; - } - if ( type==F32_TYPE ) { - if(setValue.f32 < 0) - predValue.u64 |= 1<<1; - } +#define my_abs(a) (((a) < 0) ? (-a) : (a)) - if(dst.get_operand_lohi() == 1) - { - setValue.u64 = ((m_regs.back()[ regName ].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); - } - else if(dst.get_operand_lohi() == 2) - { - setValue.u64 = ((m_regs.back()[ regName ].u64) & (~(0xFFFF0000))) + ((data.u64<<16) & 0xFFFF0000); - } +#define MY_MAX_I(a, b) (a > b) ? a : b +#define MY_MAX_F(a, b) isNaN(a) ? b : isNaN(b) ? a : (a > b) ? a : b - set_reg(predName,predValue); - set_reg(regName,setValue); - } - else if (type == BB128_TYPE) - { - //b128 stuff here. - ptx_reg_t setValue2, setValue3, setValue4; - setValue.u64 = 0; - setValue2.u64 = 0; - setValue3.u64 = 0; - setValue4.u64 = 0; - setValue.u32 = data.u128.lowest; - setValue2.u32 = data.u128.low; - setValue3.u32 = data.u128.high; - setValue4.u32 = data.u128.highest; - - const symbol *name1, *name2, *name3, *name4 = NULL; - - name1 = dst.vec_symbol(0); - name2 = dst.vec_symbol(1); - name3 = dst.vec_symbol(2); - name4 = dst.vec_symbol(3); - - set_reg(name1,setValue); - set_reg(name2,setValue2); - set_reg(name3,setValue3); - set_reg(name4,setValue4); - } - else if (type == BB64_TYPE || type == FF64_TYPE) - { - //ptxplus version of storing 64 bit values to registers stores to two adjacent registers - ptx_reg_t setValue2; - setValue.u32 = 0; - setValue2.u32 = 0; +#define MY_MIN_I(a, b) (a < b) ? a : b +#define MY_MIN_F(a, b) isNaN(a) ? b : isNaN(b) ? a : (a < b) ? a : b - setValue.u32 = data.bits.ls; - setValue2.u32 = data.bits.ms; +#define MY_INC_I(a, b) (a >= b) ? 0 : a + 1 +#define MY_DEC_I(a, b) ((a == 0) || (a > b)) ? b : a - 1 - const symbol *name1, *name2 = NULL; +#define MY_CAS_I(a, b, c) (a == b) ? c : a - name1 = dst.vec_symbol(0); - name2 = dst.vec_symbol(1); +#define MY_EXCH(a, b) b - set_reg(name1,setValue); - set_reg(name2,setValue2); - } - else - { - if(dst.get_operand_lohi() == 1) - { - setValue.u64 = ((m_regs.back()[ dst.get_symbol() ].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); - } - else if(dst.get_operand_lohi() == 2) - { - setValue.u64 = ((m_regs.back()[ dst.get_symbol() ].u64) & (~(0xFFFF0000))) + ((data.u64<<16) & 0xFFFF0000); - } - set_reg(dst.get_symbol(),setValue); - } - } - - // global memory - g[4], g[$r0] - else if(dst.get_addr_space() == global_space) - { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->get_global_memory(); - type_info_key::type_decode(type,size,t); - - mem->write(dstData.u32,size/8,&data.u128,thread,pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = global_space; - } - - // shared memory - s[4], s[$r0] - else if(dst.get_addr_space() == shared_space) - { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->m_shared_mem; - type_info_key::type_decode(type,size,t); - - mem->write(dstData.u32,size/8,&data.u128,thread,pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = shared_space; - } - - // local memory - l0[4], l0[$r0] - else if(dst.get_addr_space() == local_space) - { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->m_local_mem; - type_info_key::type_decode(type,size,t); - - mem->write(dstData.u32,size/8,&data.u128,thread,pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = local_space; - } - - else - { - printf("Destination stores to unknown location."); - assert(0); - } - - -} - -void ptx_thread_info::set_vector_operand_values( const operand_info &dst, - const ptx_reg_t &data1, - const ptx_reg_t &data2, - const ptx_reg_t &data3, - const ptx_reg_t &data4 ) -{ - unsigned num_elements = dst.get_vect_nelem(); - if (num_elements > 0) { - set_reg(dst.vec_symbol(0), data1); - if (num_elements > 1) { - set_reg(dst.vec_symbol(1), data2); - if (num_elements > 2) { - set_reg(dst.vec_symbol(2), data3); - if (num_elements > 3) { - set_reg(dst.vec_symbol(3), data4); - } - } - } - } - - m_last_set_operand_value = data1; -} -void ptx_thread_info::set_wmma_vector_operand_values( const operand_info &dst, - const ptx_reg_t &data1, - const ptx_reg_t &data2, - const ptx_reg_t &data3, - const ptx_reg_t &data4, - const ptx_reg_t &data5, - const ptx_reg_t &data6, - const ptx_reg_t &data7, - const ptx_reg_t &data8 ) -{ - unsigned num_elements = dst.get_vect_nelem(); - if (num_elements == 8) { - set_reg(dst.vec_symbol(0), data1); - set_reg(dst.vec_symbol(1), data2); - set_reg(dst.vec_symbol(2), data3); - set_reg(dst.vec_symbol(3), data4); - set_reg(dst.vec_symbol(4), data5); - set_reg(dst.vec_symbol(5), data6); - set_reg(dst.vec_symbol(6), data7); - set_reg(dst.vec_symbol(7), data8); - } - else{ - printf("error:set_wmma_vector_operands"); - } - - m_last_set_operand_value = data8; -} - -#define my_abs(a) (((a)<0)?(-a):(a)) - -#define MY_MAX_I(a,b) (a > b) ? a : b -#define MY_MAX_F(a,b) isNaN(a) ? b : isNaN(b) ? a : (a > b) ? a : b - -#define MY_MIN_I(a,b) (a < b) ? a : b -#define MY_MIN_F(a,b) isNaN(a) ? b : isNaN(b) ? a : (a < b) ? a : b - -#define MY_INC_I(a,b) (a >= b) ? 0 : a+1 -#define MY_DEC_I(a,b) ((a == 0) || (a > b)) ? b : a-1 - -#define MY_CAS_I(a,b,c) (a == b) ? c : a - -#define MY_EXCH(a,b) b - -void abs_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case S16_TYPE: d.s16 = my_abs(a.s16); break; - case S32_TYPE: d.s32 = my_abs(a.s32); break; - case S64_TYPE: d.s64 = my_abs(a.s64); break; - case U16_TYPE: d.s16 = my_abs(a.u16); break; - case U32_TYPE: d.s32 = my_abs(a.u32); break; - case U64_TYPE: d.s64 = my_abs(a.u64); break; - case F32_TYPE: d.f32 = my_abs(a.f32); break; - case F64_TYPE: case FF64_TYPE: d.f64 = my_abs(a.f64); break; - default: +void abs_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch (i_type) { + case S16_TYPE: + d.s16 = my_abs(a.s16); + break; + case S32_TYPE: + d.s32 = my_abs(a.s32); + break; + case S64_TYPE: + d.s64 = my_abs(a.s64); + break; + case U16_TYPE: + d.s16 = my_abs(a.u16); + break; + case U32_TYPE: + d.s32 = my_abs(a.u32); + break; + case U64_TYPE: + d.s64 = my_abs(a.u64); + break; + case F32_TYPE: + d.f32 = my_abs(a.f32); + break; + case F64_TYPE: + case FF64_TYPE: + d.f64 = my_abs(a.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } - - thread->set_operand_value(dst,d, i_type, thread, pI); -} - -void addp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - //PTXPlus add instruction with carry (carry is kept in a predicate) register - ptx_reg_t src1_data, src2_data, src3_data, data; - int overflow = 0; - int carry = 0; - - const operand_info &dst = pI->dst(); //get operand info of sources and destination - const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - - unsigned rounding_mode = pI->rounding_mode(); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - - //performs addition. Sets carry and overflow if needed. - //src3_data.pred&0x4 is the carry flag - switch ( i_type ) { - case S8_TYPE: - data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF) + (src3_data.pred & 0x4); - if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) == 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } - carry = (data.u64 & 0x000000100)>>8; - break; - case S16_TYPE: - data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF) + (src3_data.pred & 0x4); - if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) == 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } - carry = (data.u64 & 0x000010000)>>16; - break; - case S32_TYPE: - data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF) + (src3_data.pred & 0x4); - if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) == 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } - carry = (data.u64 & 0x100000000)>>32; - break; - case S64_TYPE: + } + + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +void addp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // PTXPlus add instruction with carry (carry is kept in a predicate) register + ptx_reg_t src1_data, src2_data, src3_data, data; + int overflow = 0; + int carry = 0; + + const operand_info &dst = + pI->dst(); // get operand info of sources and destination + const operand_info &src1 = + pI->src1(); // use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + + unsigned rounding_mode = pI->rounding_mode(); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + + // performs addition. Sets carry and overflow if needed. + // src3_data.pred&0x4 is the carry flag + switch (i_type) { + case S8_TYPE: + data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF) + + (src3_data.pred & 0x4); + if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) == 0) { + overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; + } + carry = (data.u64 & 0x000000100) >> 8; + break; + case S16_TYPE: + data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF) + + (src3_data.pred & 0x4); + if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) == 0) { + overflow = + ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; + } + carry = (data.u64 & 0x000010000) >> 16; + break; + case S32_TYPE: + data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF) + + (src3_data.pred & 0x4); + if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) == 0) { + overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 + ? 0 + : 1; + } + carry = (data.u64 & 0x100000000) >> 32; + break; + case S64_TYPE: data.s64 = src1_data.s64 + src2_data.s64 + (src3_data.pred & 0x4); break; - case U8_TYPE: - data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF) + (src3_data.pred & 0x4); - carry = (data.u64 & 0x100)>>8; + case U8_TYPE: + data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF) + + (src3_data.pred & 0x4); + carry = (data.u64 & 0x100) >> 8; break; - case U16_TYPE: - data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF) + (src3_data.pred & 0x4); - carry = (data.u64 & 0x10000)>>16; + case U16_TYPE: + data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF) + + (src3_data.pred & 0x4); + carry = (data.u64 & 0x10000) >> 16; break; - case U32_TYPE: - data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF) + (src3_data.pred & 0x4); - carry = (data.u64 & 0x100000000)>>32; + case U32_TYPE: + data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF) + + (src3_data.pred & 0x4); + carry = (data.u64 & 0x100000000) >> 32; break; - case U64_TYPE: + case U64_TYPE: data.s64 = src1_data.s64 + src2_data.s64 + (src3_data.pred & 0x4); break; - case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; - default: assert(0); break; - } - fesetround( orig_rm ); - - thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry ); -} - -void add_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - int overflow = 0; - int carry = 0; - - const operand_info &dst = pI->dst(); //get operand info of sources and destination - const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - - unsigned rounding_mode = pI->rounding_mode(); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - - //performs addition. Sets carry and overflow if needed. - switch ( i_type ) { - case S8_TYPE: + case F16_TYPE: + data.f16 = src1_data.f16 + src2_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = src1_data.f32 + src2_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = src1_data.f64 + src2_data.f64; + break; + default: + assert(0); + break; + } + fesetround(orig_rm); + + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); +} + +void add_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + int overflow = 0; + int carry = 0; + + const operand_info &dst = + pI->dst(); // get operand info of sources and destination + const operand_info &src1 = + pI->src1(); // use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + unsigned rounding_mode = pI->rounding_mode(); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + + // performs addition. Sets carry and overflow if needed. + switch (i_type) { + case S8_TYPE: data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF); - if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) == 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } - carry = (data.u64 & 0x000000100)>>8; + if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) == 0) { + overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; + } + carry = (data.u64 & 0x000000100) >> 8; break; - case S16_TYPE: + case S16_TYPE: data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF); - if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) == 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } - carry = (data.u64 & 0x000010000)>>16; + if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) == 0) { + overflow = + ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; + } + carry = (data.u64 & 0x000010000) >> 16; break; - case S32_TYPE: + case S32_TYPE: data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF); - if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) == 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } - carry = (data.u64 & 0x100000000)>>32; + if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) == 0) { + overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 + ? 0 + : 1; + } + carry = (data.u64 & 0x100000000) >> 32; break; - case S64_TYPE: + case S64_TYPE: data.s64 = src1_data.s64 + src2_data.s64; break; - case U8_TYPE: + case U8_TYPE: data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF); - carry = (data.u64 & 0x100)>>8; + carry = (data.u64 & 0x100) >> 8; break; - case U16_TYPE: + case U16_TYPE: data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF); - carry = (data.u64 & 0x10000)>>16; + carry = (data.u64 & 0x10000) >> 16; break; - case U32_TYPE: + case U32_TYPE: data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF); - carry = (data.u64 & 0x100000000)>>32; + carry = (data.u64 & 0x100000000) >> 32; break; - case U64_TYPE: + case U64_TYPE: data.u64 = src1_data.u64 + src2_data.u64; break; - case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; - default: assert(0); break; - } - fesetround( orig_rm ); + case F16_TYPE: + data.f16 = src1_data.f16 + src2_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = src1_data.f32 + src2_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = src1_data.f64 + src2_data.f64; + break; + default: + assert(0); + break; + } + fesetround(orig_rm); - thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry ); + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); } -void addc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } - -void and_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +void addc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void and_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) & ~(src2_data.pred)); - else - data.u64 = src1_data.u64 & src2_data.u64; + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) & ~(src2_data.pred)); + else + data.u64 = src1_data.u64 & src2_data.u64; - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void andn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +void andn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case B16_TYPE: src2_data.u16 = ~src2_data.u16; break; - case B32_TYPE: src2_data.u32 = ~src2_data.u32; break; - case B64_TYPE: src2_data.u64 = ~src2_data.u64; break; - default: + switch (i_type) { + case B16_TYPE: + src2_data.u16 = ~src2_data.u16; + break; + case B32_TYPE: + src2_data.u32 = ~src2_data.u32; + break; + case B64_TYPE: + src2_data.u64 = ~src2_data.u64; + break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - - data.u64 = src1_data.u64 & src2_data.u64; - - thread->set_operand_value(dst,data, i_type, thread, pI); -} - -void bar_callback( const inst_t* inst, ptx_thread_info* thread) -{ - unsigned ctaid = thread->get_cta_uid(); - unsigned barid = inst->bar_id; - unsigned value = thread->get_reduction_value(ctaid,barid); - const ptx_instruction *pI = dynamic_cast(inst); - const operand_info &dst = pI->dst(); - ptx_reg_t data; - data.u32 = value; - thread->set_operand_value(dst,value, U32_TYPE, thread, pI); -} - -void atom_callback( const inst_t* inst, ptx_thread_info* thread) -{ - const ptx_instruction *pI = dynamic_cast(inst); - - // "Decode" the output type - unsigned to_type = pI->get_type(); - size_t size; - int t; - type_info_key::type_decode(to_type, size, t); - - // Set up operand variables - ptx_reg_t data; // d - ptx_reg_t src1_data; // a - ptx_reg_t src2_data; // b - ptx_reg_t op_result; // temp variable to hold operation result - - bool data_ready = false; - - // Get operand info of sources and destination - const operand_info &dst = pI->dst(); // d - const operand_info &src1 = pI->src1(); // a - const operand_info &src2 = pI->src2(); // b - - // Get operand values - src1_data = thread->get_operand_value(src1, src1, to_type, thread, 1); // a - if (dst.get_symbol()->type()){ - src2_data = thread->get_operand_value(src2, dst, to_type, thread, 1); // b - } else { - //This is the case whent he first argument (dest) is '_' - src2_data = thread->get_operand_value(src2, src1, to_type, thread, 1); // b - } - - // Check state space - addr_t effective_address = src1_data.u64; - memory_space_t space = pI->get_space(); - if (space == undefined_space) { - // generic space - determine space via address - if( whichspace(effective_address) == global_space ) { - effective_address = generic_to_global(effective_address); - space = global_space; - } else if( whichspace(effective_address) == shared_space ) { - unsigned smid = thread->get_hw_sid(); - effective_address = generic_to_shared(smid,effective_address); - space = shared_space; - } else { - abort(); - } - } - assert( space == global_space || space == shared_space ); - - memory_space *mem = NULL; - if(space == global_space) - mem = thread->get_global_memory(); - else if(space == shared_space) - mem = thread->m_shared_mem; - else - abort(); - - // Copy value pointed to in operand 'a' into register 'd' - // (i.e. copy src1_data to dst) - mem->read(effective_address,size/8,&data.s64); - if (dst.get_symbol()->type()){ - thread->set_operand_value(dst, data, to_type, thread, pI); // Write value into register 'd' - } - - // Get the atomic operation to be performed - unsigned m_atomic_spec = pI->get_atomic(); - - switch ( m_atomic_spec ) { - // AND - case ATOMIC_AND: - { - - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 & src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 & src2_data.s32; - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.AND only accepts b32\n", to_type); - assert(0); - break; - } + assert(0); + break; + } - break; - } - // OR - case ATOMIC_OR: - { - - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 | src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 | src2_data.s32; - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.OR only accepts b32\n", to_type); - assert(0); - break; - } + data.u64 = src1_data.u64 & src2_data.u64; - break; - } - // XOR - case ATOMIC_XOR: - { - - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 ^ src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 ^ src2_data.s32; - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.XOR only accepts b32\n", to_type); - assert(0); - break; - } + thread->set_operand_value(dst, data, i_type, thread, pI); +} - break; - } - // CAS - case ATOMIC_CAS: - { - - ptx_reg_t src3_data; - const operand_info &src3 = pI->src3(); - src3_data = thread->get_operand_value(src3, dst, to_type, thread, 1); - - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = MY_CAS_I(data.u32, src2_data.u32, src3_data.u32); - data_ready = true; - break; - case B64_TYPE: - case U64_TYPE: - op_result.u64 = MY_CAS_I(data.u64, src2_data.u64, src3_data.u64); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_CAS_I(data.s32, src2_data.s32, src3_data.s32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.CAS only accepts b32 and b64\n", to_type); - assert(0); - break; - } +void bar_callback(const inst_t *inst, ptx_thread_info *thread) { + unsigned ctaid = thread->get_cta_uid(); + unsigned barid = inst->bar_id; + unsigned value = thread->get_reduction_value(ctaid, barid); + const ptx_instruction *pI = dynamic_cast(inst); + const operand_info &dst = pI->dst(); + ptx_reg_t data; + data.u32 = value; + thread->set_operand_value(dst, value, U32_TYPE, thread, pI); +} - break; - } - // EXCH - case ATOMIC_EXCH: - { - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = MY_EXCH(data.u32, src2_data.u32); - data_ready = true; - break; - case B64_TYPE: - case U64_TYPE: - op_result.u64 = MY_EXCH(data.u64, src2_data.u64); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_EXCH(data.s32, src2_data.s32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.EXCH only accepts b32\n", to_type); - assert(0); - break; - } +void atom_callback(const inst_t *inst, ptx_thread_info *thread) { + const ptx_instruction *pI = dynamic_cast(inst); + + // "Decode" the output type + unsigned to_type = pI->get_type(); + size_t size; + int t; + type_info_key::type_decode(to_type, size, t); + + // Set up operand variables + ptx_reg_t data; // d + ptx_reg_t src1_data; // a + ptx_reg_t src2_data; // b + ptx_reg_t op_result; // temp variable to hold operation result + + bool data_ready = false; + + // Get operand info of sources and destination + const operand_info &dst = pI->dst(); // d + const operand_info &src1 = pI->src1(); // a + const operand_info &src2 = pI->src2(); // b + + // Get operand values + src1_data = thread->get_operand_value(src1, src1, to_type, thread, 1); // a + if (dst.get_symbol()->type()) { + src2_data = thread->get_operand_value(src2, dst, to_type, thread, 1); // b + } else { + // This is the case whent he first argument (dest) is '_' + src2_data = thread->get_operand_value(src2, src1, to_type, thread, 1); // b + } + + // Check state space + addr_t effective_address = src1_data.u64; + memory_space_t space = pI->get_space(); + if (space == undefined_space) { + // generic space - determine space via address + if (whichspace(effective_address) == global_space) { + effective_address = generic_to_global(effective_address); + space = global_space; + } else if (whichspace(effective_address) == shared_space) { + unsigned smid = thread->get_hw_sid(); + effective_address = generic_to_shared(smid, effective_address); + space = shared_space; + } else { + abort(); + } + } + assert(space == global_space || space == shared_space); + + memory_space *mem = NULL; + if (space == global_space) + mem = thread->get_global_memory(); + else if (space == shared_space) + mem = thread->m_shared_mem; + else + abort(); + + // Copy value pointed to in operand 'a' into register 'd' + // (i.e. copy src1_data to dst) + mem->read(effective_address, size / 8, &data.s64); + if (dst.get_symbol()->type()) { + thread->set_operand_value(dst, data, to_type, thread, + pI); // Write value into register 'd' + } + + // Get the atomic operation to be performed + unsigned m_atomic_spec = pI->get_atomic(); - break; + switch (m_atomic_spec) { + // AND + case ATOMIC_AND: { + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 & src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 & src2_data.s32; + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.AND " + "only accepts b32\n", + to_type); + assert(0); + break; + } + + break; + } + // OR + case ATOMIC_OR: { + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 | src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 | src2_data.s32; + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.OR " + "only accepts b32\n", + to_type); + assert(0); + break; } - // ADD - case ATOMIC_ADD: - { - - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = data.u32 + src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 + src2_data.s32; - data_ready = true; - break; - case U64_TYPE: - op_result.u64 = data.u64 + src2_data.u64; - data_ready = true; - break; - case F32_TYPE: - op_result.f32 = data.f32 + src2_data.f32; - data_ready = true; - break; - default: - printf("Execution error: type mismatch with instruction\natom.ADD only accepts u32, s32, u64, and f32\n"); - assert(0); - break; - } - break; + break; + } + // XOR + case ATOMIC_XOR: { + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 ^ src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 ^ src2_data.s32; + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.XOR " + "only accepts b32\n", + to_type); + assert(0); + break; } - // INC - case ATOMIC_INC: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = MY_INC_I(data.u32, src2_data.u32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch with instruction\natom.INC only accepts u32 and s32\n"); - assert(0); - break; - } - break; + break; + } + // CAS + case ATOMIC_CAS: { + ptx_reg_t src3_data; + const operand_info &src3 = pI->src3(); + src3_data = thread->get_operand_value(src3, dst, to_type, thread, 1); + + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = MY_CAS_I(data.u32, src2_data.u32, src3_data.u32); + data_ready = true; + break; + case B64_TYPE: + case U64_TYPE: + op_result.u64 = MY_CAS_I(data.u64, src2_data.u64, src3_data.u64); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_CAS_I(data.s32, src2_data.s32, src3_data.s32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.CAS " + "only accepts b32 and b64\n", + to_type); + assert(0); + break; } - // DEC - case ATOMIC_DEC: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = MY_DEC_I(data.u32, src2_data.u32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch with instruction\natom.DEC only accepts u32 and s32\n"); - assert(0); - break; - } - break; + break; + } + // EXCH + case ATOMIC_EXCH: { + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = MY_EXCH(data.u32, src2_data.u32); + data_ready = true; + break; + case B64_TYPE: + case U64_TYPE: + op_result.u64 = MY_EXCH(data.u64, src2_data.u64); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_EXCH(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.EXCH " + "only accepts b32\n", + to_type); + assert(0); + break; + } + + break; + } + // ADD + case ATOMIC_ADD: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = data.u32 + src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 + src2_data.s32; + data_ready = true; + break; + case U64_TYPE: + op_result.u64 = data.u64 + src2_data.u64; + data_ready = true; + break; + case F32_TYPE: + op_result.f32 = data.f32 + src2_data.f32; + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.ADD only " + "accepts u32, s32, u64, and f32\n"); + assert(0); + break; + } + + break; + } + // INC + case ATOMIC_INC: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = MY_INC_I(data.u32, src2_data.u32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.INC only " + "accepts u32 and s32\n"); + assert(0); + break; + } + + break; + } + // DEC + case ATOMIC_DEC: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = MY_DEC_I(data.u32, src2_data.u32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.DEC only " + "accepts u32 and s32\n"); + assert(0); + break; + } + + break; + } + // MIN + case ATOMIC_MIN: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = MY_MIN_I(data.u32, src2_data.u32); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_MIN_I(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.MIN only " + "accepts u32 and s32\n"); + assert(0); + break; + } + + break; + } + // MAX + case ATOMIC_MAX: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = MY_MAX_I(data.u32, src2_data.u32); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_MAX_I(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.MAX only " + "accepts u32 and s32\n"); + assert(0); + break; + } + + break; + } + // DEFAULT + default: { + assert(0); + break; + } + } + + // Write operation result into memory + // (i.e. copy src1_data to dst) + if (data_ready) { + mem->write(effective_address, size / 8, &op_result.s64, thread, pI); + } else { + printf("Execution error: data_ready not set\n"); + assert(0); + } +} + +// atom_impl will now result in a callback being called in mem_ctrl_pop +// (gpu-sim.c) +void atom_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // SYNTAX + // atom.space.operation.type d, a, b[, c]; (now read in callback) + + // obtain memory space of the operation + memory_space_t space = pI->get_space(); + + // get the memory address + const operand_info &src1 = pI->src1(); + // const operand_info &dst = pI->dst(); // not needed for effective address + // calculation + unsigned i_type = pI->get_type(); + ptx_reg_t src1_data; + src1_data = thread->get_operand_value(src1, src1, i_type, thread, 1); + addr_t effective_address = src1_data.u64; + + addr_t effective_address_final; + + // handle generic memory space by converting it to global + if (space == undefined_space) { + if (whichspace(effective_address) == global_space) { + effective_address_final = generic_to_global(effective_address); + space = global_space; + } else if (whichspace(effective_address) == shared_space) { + unsigned smid = thread->get_hw_sid(); + effective_address_final = generic_to_shared(smid, effective_address); + space = shared_space; + } else { + abort(); + } + } else { + assert(space == global_space || space == shared_space); + effective_address_final = effective_address; + } + + // Check state space + assert(space == global_space || space == shared_space); + + thread->m_last_effective_address = effective_address_final; + thread->m_last_memory_space = space; + thread->m_last_dram_callback.function = atom_callback; + thread->m_last_dram_callback.instruction = pI; +} + +void bar_impl(const ptx_instruction *pIin, ptx_thread_info *thread) { + ptx_instruction *pI = const_cast(pIin); + unsigned bar_op = pI->barrier_op(); + unsigned red_op = pI->get_atomic(); + unsigned ctaid = thread->get_cta_uid(); + + switch (bar_op) { + case SYNC_OPTION: { + if (pI->get_num_operands() > 1) { + const operand_info &op0 = pI->dst(); + const operand_info &op1 = pI->src1(); + ptx_reg_t op0_data; + ptx_reg_t op1_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + pI->set_bar_count(op1_data.u32); + } else { + const operand_info &op0 = pI->dst(); + ptx_reg_t op0_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); } - // MIN - case ATOMIC_MIN: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = MY_MIN_I(data.u32, src2_data.u32); - data_ready = true; + break; + } + case ARRIVE_OPTION: { + const operand_info &op0 = pI->dst(); + const operand_info &op1 = pI->src1(); + ptx_reg_t op0_data; + ptx_reg_t op1_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + pI->set_bar_count(op1_data.u32); + break; + } + case RED_OPTION: { + if (pI->get_num_operands() > 3) { + const operand_info &op1 = pI->src1(); + const operand_info &op2 = pI->src2(); + const operand_info &op3 = pI->src3(); + ptx_reg_t op1_data; + ptx_reg_t op2_data; + ptx_reg_t op3_data; + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + op2_data = thread->get_operand_value(op2, op2, U32_TYPE, thread, 1); + op3_data = thread->get_operand_value(op3, op3, PRED_TYPE, thread, 1); + op3_data.u32 = !(op3_data.pred & 0x0001); + pI->set_bar_id(op1_data.u32); + pI->set_bar_count(op2_data.u32); + switch (red_op) { + case ATOMIC_POPC: + thread->popc_reduction(ctaid, op1_data.u32, op3_data.u32); break; - case S32_TYPE: - op_result.s32 = MY_MIN_I(data.s32, src2_data.s32); - data_ready = true; + case ATOMIC_AND: + thread->and_reduction(ctaid, op1_data.u32, op3_data.u32); break; - default: - printf("Execution error: type mismatch with instruction\natom.MIN only accepts u32 and s32\n"); - assert(0); + case ATOMIC_OR: + thread->or_reduction(ctaid, op1_data.u32, op3_data.u32); break; - } - - break; - } - // MAX - case ATOMIC_MAX: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = MY_MAX_I(data.u32, src2_data.u32); - data_ready = true; + default: + abort(); break; - case S32_TYPE: - op_result.s32 = MY_MAX_I(data.s32, src2_data.s32); - data_ready = true; + } + } else { + const operand_info &op1 = pI->src1(); + const operand_info &op2 = pI->src2(); + ptx_reg_t op1_data; + ptx_reg_t op2_data; + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + op2_data = thread->get_operand_value(op2, op2, PRED_TYPE, thread, 1); + op2_data.u32 = !(op2_data.pred & 0x0001); + pI->set_bar_id(op1_data.u32); + pI->set_bar_count(thread->get_ntid().x * thread->get_ntid().y * + thread->get_ntid().z); + switch (red_op) { + case ATOMIC_POPC: + thread->popc_reduction(ctaid, op1_data.u32, op2_data.u32); break; - default: - printf("Execution error: type mismatch with instruction\natom.MAX only accepts u32 and s32\n"); - assert(0); + case ATOMIC_AND: + thread->and_reduction(ctaid, op1_data.u32, op2_data.u32); break; - } - - break; + case ATOMIC_OR: + thread->or_reduction(ctaid, op1_data.u32, op2_data.u32); + break; + default: + abort(); + break; + } } - // DEFAULT - default: - { - assert(0); - break; + break; + } + default: + abort(); + break; + } + + thread->m_last_dram_callback.function = bar_callback; + thread->m_last_dram_callback.instruction = pIin; +} + +void bfe_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + unsigned i_type = pI->get_type(); + unsigned msb = (i_type == U32_TYPE || i_type == S32_TYPE) ? 31 : 63; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t src = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + ptx_reg_t data; + unsigned pos = b.u32 & 0xFF; + unsigned len = c.u32 & 0xFF; + switch (i_type) { + case U32_TYPE: { + unsigned mask; + data.u32 = src.u32 >> pos; + mask = 0xFFFFFFFF >> (32 - len); + data.u32 &= mask; + break; + } + case U64_TYPE: { + unsigned long mask; + data.u64 = src.u64 >> pos; + mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); + data.u64 &= mask; + break; + } + case S32_TYPE: { + unsigned mask; + unsigned min = MY_MIN_I(pos + len - 1, msb); + unsigned sbit = len == 0 ? 0 : (src.s32 >> min) & 0x1; + data.s32 = src.s32 >> pos; + if (sbit > 0) { + mask = 0xFFFFFFFF << len; + data.s32 |= mask; + } else { + mask = 0xFFFFFFFF >> (32 - len); + data.s32 &= mask; } - } - - // Write operation result into memory - // (i.e. copy src1_data to dst) - if ( data_ready ) { - mem->write(effective_address,size/8,&op_result.s64,thread,pI); - } else { - printf("Execution error: data_ready not set\n"); - assert(0); - } -} - -// atom_impl will now result in a callback being called in mem_ctrl_pop (gpu-sim.c) -void atom_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - // SYNTAX - // atom.space.operation.type d, a, b[, c]; (now read in callback) - - // obtain memory space of the operation - memory_space_t space = pI->get_space(); - - // get the memory address - const operand_info &src1 = pI->src1(); - // const operand_info &dst = pI->dst(); // not needed for effective address calculation - unsigned i_type = pI->get_type(); - ptx_reg_t src1_data; - src1_data = thread->get_operand_value(src1, src1, i_type, thread, 1); - addr_t effective_address = src1_data.u64; - - addr_t effective_address_final; - - // handle generic memory space by converting it to global - if ( space == undefined_space ) { - if( whichspace(effective_address) == global_space ) { - effective_address_final = generic_to_global(effective_address); - space = global_space; - } else if( whichspace(effective_address) == shared_space ) { - unsigned smid = thread->get_hw_sid(); - effective_address_final = generic_to_shared(smid,effective_address); - space = shared_space; + break; + } + case S64_TYPE: { + unsigned long mask; + unsigned min = MY_MIN_I(pos + len - 1, msb); + unsigned sbit = len == 0 ? 0 : (src.s64 >> min) & 0x1; + data.s64 = src.s64 >> pos; + if (sbit > 0) { + mask = 0xFFFFFFFFFFFFFFFF << len; + data.s64 |= mask; } else { - abort(); + mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); + data.s64 &= mask; } - } else { - assert( space == global_space || space == shared_space ); - effective_address_final = effective_address; - } - - // Check state space - assert( space == global_space || space == shared_space ); - - thread->m_last_effective_address = effective_address_final; - thread->m_last_memory_space = space; - thread->m_last_dram_callback.function = atom_callback; - thread->m_last_dram_callback.instruction = pI; -} - -void bar_impl( const ptx_instruction *pIin, ptx_thread_info *thread ) -{ - ptx_instruction * pI = const_cast(pIin); - unsigned bar_op = pI->barrier_op(); - unsigned red_op = pI->get_atomic(); - unsigned ctaid = thread->get_cta_uid(); - - switch(bar_op){ - case SYNC_OPTION: - { - if(pI->get_num_operands()>1){ - const operand_info &op0 = pI->dst(); - const operand_info &op1 = pI->src1(); - ptx_reg_t op0_data; - ptx_reg_t op1_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); - pI->set_bar_count(op1_data.u32); - }else{ - const operand_info &op0 = pI->dst(); - ptx_reg_t op0_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); - } - break; - } - case ARRIVE_OPTION: - { - const operand_info &op0 = pI->dst(); - const operand_info &op1 = pI->src1(); - ptx_reg_t op0_data; - ptx_reg_t op1_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); - pI->set_bar_count(op1_data.u32); - break; - } - case RED_OPTION: - { - if(pI->get_num_operands()>3){ - const operand_info &op1 = pI->src1(); - const operand_info &op2 = pI->src2(); - const operand_info &op3 = pI->src3(); - ptx_reg_t op1_data; - ptx_reg_t op2_data; - ptx_reg_t op3_data; - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - op2_data = thread->get_operand_value(op2, op2, U32_TYPE, thread, 1); - op3_data = thread->get_operand_value(op3, op3, PRED_TYPE, thread, 1); - op3_data.u32=!(op3_data.pred & 0x0001); - pI->set_bar_id(op1_data.u32); - pI->set_bar_count(op2_data.u32); - switch(red_op){ - case ATOMIC_POPC: - thread->popc_reduction(ctaid,op1_data.u32,op3_data.u32); - break; - case ATOMIC_AND: - thread->and_reduction(ctaid,op1_data.u32,op3_data.u32); - break; - case ATOMIC_OR: - thread->or_reduction(ctaid,op1_data.u32,op3_data.u32); - break; - default: - abort(); - break; - } - }else{ - const operand_info &op1 = pI->src1(); - const operand_info &op2 = pI->src2(); - ptx_reg_t op1_data; - ptx_reg_t op2_data; - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - op2_data = thread->get_operand_value(op2, op2, PRED_TYPE, thread, 1); - op2_data.u32=!(op2_data.pred & 0x0001); - pI->set_bar_id(op1_data.u32); - pI->set_bar_count(thread->get_ntid().x * thread->get_ntid().y * thread->get_ntid().z); - switch(red_op){ - case ATOMIC_POPC: - thread->popc_reduction(ctaid,op1_data.u32,op2_data.u32); - break; - case ATOMIC_AND: - thread->and_reduction(ctaid,op1_data.u32,op2_data.u32); - break; - case ATOMIC_OR: - thread->or_reduction(ctaid,op1_data.u32,op2_data.u32); - break; - default: - abort(); - break; - } - } - break; - } - default: - abort(); - break; - } - - thread->m_last_dram_callback.function = bar_callback; - thread->m_last_dram_callback.instruction = pIin; -} - -void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - unsigned i_type = pI->get_type(); - unsigned msb = (i_type == U32_TYPE || i_type == S32_TYPE) ? 31 : 63; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t src = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - ptx_reg_t data; - unsigned pos = b.u32 & 0xFF; - unsigned len = c.u32 & 0xFF; - switch (i_type) - { - case U32_TYPE: - { - unsigned mask; - data.u32 = src.u32 >> pos; - mask = 0xFFFFFFFF >> (32 - len); - data.u32 &= mask; - break; - } - case U64_TYPE: - { - unsigned long mask; - data.u64 = src.u64 >> pos; - mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - data.u64 &= mask; - break; - } - case S32_TYPE: - { - unsigned mask; - unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (src.s32 >> min) & 0x1; - data.s32 = src.s32 >> pos; - if (sbit > 0) - { - mask = 0xFFFFFFFF << len; - data.s32 |= mask; - } - else - { - mask = 0xFFFFFFFF >> (32 - len); - data.s32 &= mask; - } - break; - } - case S64_TYPE: - { - unsigned long mask; - unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (src.s64 >> min) & 0x1; - data.s64 = src.s64 >> pos; - if (sbit > 0) - { - mask = 0xFFFFFFFFFFFFFFFF << len; - data.s64 |= mask; - } - else - { - mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - data.s64 &= mask; - } - break; - } - default: - printf("Operand type not supported for BFE instruction.\n"); - abort(); - return; - } - thread->set_operand_value(dst, data, i_type, thread, pI); + break; + } + default: + printf("Operand type not supported for BFE instruction.\n"); + abort(); + return; + } + thread->set_operand_value(dst, data, i_type, thread, pI); } -void bfi_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { - int i,max; - ptx_reg_t src1_data, src2_data; - ptx_reg_t src3_data, src4_data, data; - - const operand_info &dst = pI->dst(); //get operand info of sources and destination - const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - const operand_info &src4 = pI->src4(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - src4_data = thread->get_operand_value(src4, dst, i_type, thread, 1); - - switch ( i_type ) { - case B32_TYPE: +void bfi_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + int i, max; + ptx_reg_t src1_data, src2_data; + ptx_reg_t src3_data, src4_data, data; + + const operand_info &dst = + pI->dst(); // get operand info of sources and destination + const operand_info &src1 = + pI->src1(); // use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + const operand_info &src4 = pI->src4(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + src4_data = thread->get_operand_value(src4, dst, i_type, thread, 1); + + switch (i_type) { + case B32_TYPE: max = 32; break; - case B64_TYPE: + case B64_TYPE: max = 64; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } - data=src2_data; - unsigned pos = src3_data.u32 & 0xFF; - unsigned len = src4_data.u32 & 0xFF; - for(i=0;iset_operand_value(dst, data, i_type, thread, pI); + } + data = src2_data; + unsigned pos = src3_data.u32 & 0xFF; + unsigned len = src4_data.u32 & 0xFF; + for (i = 0; i < len && pos + i < max; i++) { + data.u32 = (~((0x00000001) << (pos + i))) & data.u32; + data.u32 = data.u32 | ((src1_data.u32 & ((0x00000001) << (i))) << (pos)); + } + thread->set_operand_value(dst, data, i_type, thread, pI); +} +void bfind_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} + +void bra_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = + thread->get_operand_value(target, target, U32_TYPE, thread, 1); + + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void bfind_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void bra_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void brx_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = + thread->get_operand_value(target, target, U32_TYPE, thread, 1); - thread->m_branch_taken = true; - thread->set_npc(target_pc); + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void brx_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void break_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &target = thread->pop_breakaddr(); + ptx_reg_t target_pc = + thread->get_operand_value(target, target, U32_TYPE, thread, 1); - thread->m_branch_taken = true; - thread->set_npc(target_pc); + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void break_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &target = thread->pop_breakaddr(); - ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void breakaddr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &target = pI->dst(); + thread->push_breakaddr(target); + assert( + pI->has_pred() == + false); // pdom analysis cannot handle if this instruction is predicated +} - thread->m_branch_taken = true; - thread->set_npc(target_pc); +void brev_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + unsigned msb; + switch (i_type) { + case B32_TYPE: + msb = 31; + for (unsigned i = 0; i <= msb; i++) { + if ((src1_data.u32 & (1 << i))) data.u32 |= 1 << (msb - i); + } + break; + case B64_TYPE: + msb = 63; + for (unsigned i = 0; i <= msb; i++) { + if ((src1_data.u64 & (1 << i))) data.u64 |= 1 << (msb - i); + } + break; + default: + assert(0); + } + thread->set_operand_value(dst, data, i_type, thread, pI); +} +void brkpt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); } -void breakaddr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &target = pI->dst(); - thread->push_breakaddr(target); - assert(pI->has_pred() == false); // pdom analysis cannot handle if this instruction is predicated +unsigned trunc(unsigned num, unsigned precision) { + int mask = 1, latest_one = -1; + unsigned data = num; + for (unsigned j = 0; j < sizeof(unsigned) * 8; j++) { + int bit = data & mask; + if (bit == 1) latest_one = j; + data >>= 1; + } + if (latest_one >= precision) { + // round_up is 1 if the most significant truncated digit is a 1, otherwise + // it is 0 + // int round_up = (num & (1 << (latest_one-precision))) >> + // (latest_one-precision); + // unsigned shifted_output = num >> (latest_one-precision+1); + // if shifted_output is a number like 1111, don't round up + // if (shifted_output == (pow(2,precision)-1)) round_up = 0; + // num = shifted_output + round_up; + num >>= (latest_one - precision + 1); + } + return num; } +void mapping(int thread, int wmma_type, int wmma_layout, int type, int index, + int stride, int &row, int &col, int &assg_offset) { + int offset; + int c_row_offset[] = {0, 8, 0, 8, 4, 12, 4, 12}; + int c_col_offset[] = {0, 0, 8, 8, 0, 0, 8, 8}; + int c_tg_inside_row_offset[] = {0, 1, 0, 1}; + int c_tg_inside_col_offset[] = {0, 0, 2, 2}; + int c_inside_row_offset[] = {0, 0, 2, 2, 0, 0, 2, 2}; + int c_inside_col_offset[] = {0, 1, 0, 1, 4, 5, 4, 5}; + + offset = thread_group_offset(thread, wmma_type, wmma_layout, type, stride); + + if (wmma_type == LOAD_A) { + if (wmma_layout == ROW) { + offset += index + 8 * ((thread % 16) / 8); + } else { + offset += 64 * (index / 4) + index % 4 + 128 * ((thread % 16) / 8); + } + offset = (offset / 16) * stride + offset % 16; + assg_offset = index + 8 * ((thread % 16) / 8); + } else if (wmma_type == LOAD_B) { + if (wmma_layout == ROW) { + offset += 64 * (index / 4) + index % 4 + 128 * ((thread % 16) / 8); + } else { + offset += index + 8 * ((thread % 16) / 8); + } + offset = (offset / 16) * stride + offset % 16; + assg_offset = index + 8 * ((thread % 16) / 8); + } else if (wmma_type == LOAD_C) { + if (type == F16_TYPE) { + row = c_row_offset[thread / 4] + thread % 4; + col = c_col_offset[thread / 4] + index; + } else { + row = c_row_offset[thread / 4] + c_tg_inside_row_offset[thread % 4] + + c_inside_row_offset[index]; + col = c_col_offset[thread / 4] + c_tg_inside_col_offset[thread % 4] + + c_inside_col_offset[index]; + } + assg_offset = index; + } -void brev_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + if (wmma_type == LOAD_A || wmma_type == LOAD_B) { + if (wmma_layout == ROW) { + row = offset / 16; + col = offset % 16; + } else { + col = offset / 16; + row = offset % 16; + } + } +} - unsigned msb; - switch(i_type){ - case B32_TYPE: - msb = 31; - for (unsigned i=0; i<=msb; i++) { - if((src1_data.u32 & (1 << i))) - data.u32 |= 1 << (msb - i); - } - break; - case B64_TYPE: - msb = 63; - for (unsigned i=0; i<=msb; i++) { - if((src1_data.u64 & (1 << i))) - data.u64 |= 1 << (msb - i); +void mma_impl(const ptx_instruction *pI, core_t *core, warp_inst_t inst) { + int i, j, k, thrd; + int row, col, offset; + ptx_reg_t matrix_a[16][16]; + ptx_reg_t matrix_b[16][16]; + ptx_reg_t matrix_c[16][16]; + ptx_reg_t matrix_d[16][16]; + ptx_reg_t src_data; + ptx_thread_info *thread; + + unsigned a_layout = pI->get_wmma_layout(0); + unsigned b_layout = pI->get_wmma_layout(1); + unsigned type = pI->get_type(); + unsigned type2 = pI->get_type2(); + int tid; + const operand_info &dst = pI->operand_lookup(0); + + if (core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); + float temp; + half temp2; + + for (thrd = 0; thrd < core->get_warp_size(); thrd++) { + thread = core->get_thread_info()[tid + thrd]; + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("THREAD=%d\n:", thrd); + for (int operand_num = 1; operand_num <= 3; operand_num++) { + const operand_info &src_a = pI->operand_lookup(operand_num); + unsigned nelem = src_a.get_vect_nelem(); + ptx_reg_t v[8]; + thread->get_vector_operand_values(src_a, v, nelem); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("Thread%d_Iteration=%d\n:", thrd, operand_num); + for (k = 0; k < nelem; k++) { + printf("%llx ", v[k].u64); + } + printf("\n"); + } + ptx_reg_t nw_v[16]; + int hex_val; + + if (!((operand_num == 3) && (type2 == F32_TYPE))) { + for (k = 0; k < 2 * nelem; k++) { + if (k % 2 == 1) + hex_val = (v[k / 2].s64 & 0xffff); + else + hex_val = ((v[k / 2].s64 & 0xffff0000) >> 16); + nw_v[k].f16 = *((half *)&hex_val); + } + } + if (!((operand_num == 3) && (type2 == F32_TYPE))) { + for (k = 0; k < 2 * nelem; k++) { + temp = nw_v[k].f16; + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("%.2f ", temp); + } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) printf("\n"); + } else { + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + for (k = 0; k < 8; k++) { + printf("%.2f ", v[k].f32); + } + printf("\n"); + } + } + switch (operand_num) { + case 1: // operand 1 + for (k = 0; k < 8; k++) { + mapping(thrd, LOAD_A, a_layout, F16_TYPE, k, 16, row, col, offset); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("A:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, + offset); + matrix_a[row][col] = nw_v[offset]; + } + break; + case 2: // operand 2 + for (k = 0; k < 8; k++) { + mapping(thrd, LOAD_B, b_layout, F16_TYPE, k, 16, row, col, offset); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("B:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, + offset); + matrix_b[row][col] = nw_v[offset]; + } + break; + case 3: // operand 3 + for (k = 0; k < 8; k++) { + mapping(thrd, LOAD_C, ROW, type2, k, 16, row, col, offset); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("C:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, + offset); + if (type2 != F16_TYPE) { + matrix_c[row][col] = v[offset]; + } else { + matrix_c[row][col] = nw_v[offset]; } - break; - default: assert(0); + } + break; + default: + printf("Invalid Operand Index\n"); + } + } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) printf("\n"); + } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("MATRIX_A\n"); + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + temp = matrix_a[i][j].f16; + printf("%.2f ", temp); + } + printf("\n"); + } + printf("MATRIX_B\n"); + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + temp = matrix_b[i][j].f16; + printf("%.2f ", temp); + } + printf("\n"); + } + printf("MATRIX_C\n"); + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + if (type2 == F16_TYPE) { + temp = matrix_c[i][j].f16; + printf("%.2f ", temp); + } else + printf("%.2f ", matrix_c[i][j].f32); + } + printf("\n"); + } + } + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + matrix_d[i][j].f16 = 0; + } + } + + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + for (k = 0; k < 16; k++) { + matrix_d[i][j].f16 = + matrix_d[i][j].f16 + matrix_a[i][k].f16 * matrix_b[k][j].f16; + } + if ((type == F16_TYPE) && (type2 == F16_TYPE)) + matrix_d[i][j].f16 += matrix_c[i][j].f16; + else if ((type == F32_TYPE) && (type2 == F16_TYPE)) { + temp2 = matrix_d[i][j].f16 + matrix_c[i][j].f16; + temp = temp2; + matrix_d[i][j].f32 = temp; + } else if ((type == F16_TYPE) && (type2 == F32_TYPE)) { + temp = matrix_d[i][j].f16; + temp += matrix_c[i][j].f32; + matrix_d[i][j].f16 = half(temp); + } else { + temp = matrix_d[i][j].f16; + temp += matrix_c[i][j].f32; + matrix_d[i][j].f32 = temp; + } + } + } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("MATRIX_D\n"); + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + if (type == F16_TYPE) { + temp = matrix_d[i][j].f16; + printf("%.2f ", temp); + } else + printf("%.2f ", matrix_d[i][j].f32); + } + printf("\n"); + } + } + for (thrd = 0; thrd < core->get_warp_size(); thrd++) { + int row_t[8]; + int col_t[8]; + for (k = 0; k < 8; k++) { + mapping(thrd, LOAD_C, ROW, type, k, 16, row_t[k], col_t[k], offset); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma:store:row:%d,col%d\n", row_t[k], col_t[k]); + } + thread = core->get_thread_info()[tid + thrd]; + + if (type == F32_TYPE) { + thread->set_wmma_vector_operand_values( + dst, matrix_d[row_t[0]][col_t[0]], matrix_d[row_t[1]][col_t[1]], + matrix_d[row_t[2]][col_t[2]], matrix_d[row_t[3]][col_t[3]], + matrix_d[row_t[4]][col_t[4]], matrix_d[row_t[5]][col_t[5]], + matrix_d[row_t[6]][col_t[6]], matrix_d[row_t[7]][col_t[7]]); + + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("thread%d:", thrd); + for (k = 0; k < 8; k++) { + printf("%.2f ", matrix_d[row_t[k]][col_t[k]].f32); + } + printf("\n"); + } + } else if (type == F16_TYPE) { + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("thread%d:", thrd); + for (k = 0; k < 8; k++) { + temp = matrix_d[row_t[k]][col_t[k]].f16; + printf("%.2f ", temp); + } + printf("\n"); + + printf("thread%d:", thrd); + for (k = 0; k < 8; k++) { + printf("%x ", (unsigned int)matrix_d[row_t[k]][col_t[k]].f16); + } + printf("\n"); + } + ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4; + nw_data1.s64 = ((matrix_d[row_t[0]][col_t[0]].s64 & 0xffff)) | + ((matrix_d[row_t[1]][col_t[1]].s64 & 0xffff) << 16); + nw_data2.s64 = ((matrix_d[row_t[2]][col_t[2]].s64 & 0xffff)) | + ((matrix_d[row_t[3]][col_t[3]].s64 & 0xffff) << 16); + nw_data3.s64 = ((matrix_d[row_t[4]][col_t[4]].s64 & 0xffff)) | + ((matrix_d[row_t[5]][col_t[5]].s64 & 0xffff) << 16); + nw_data4.s64 = ((matrix_d[row_t[6]][col_t[6]].s64 & 0xffff)) | + ((matrix_d[row_t[7]][col_t[7]].s64 & 0xffff) << 16); + thread->set_vector_operand_values(dst, nw_data1, nw_data2, nw_data3, + nw_data4); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("thread%d=%llx,%llx,%llx,%llx", thrd, nw_data1.s64, nw_data2.s64, + nw_data3.s64, nw_data4.s64); + + } else { + printf("wmma:mma:wrong type\n"); + abort(); } - thread->set_operand_value(dst,data, i_type, thread, pI); + } } -void brkpt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -unsigned trunc(unsigned num, unsigned precision) { - int mask = 1, latest_one = -1; - unsigned data = num; - for (unsigned j = 0; j < sizeof(unsigned)*8; j++) { - int bit = data & mask; - if (bit == 1) latest_one = j; - data >>= 1; - } - if (latest_one >= precision) { - // round_up is 1 if the most significant truncated digit is a 1, otherwise it is 0 - //int round_up = (num & (1 << (latest_one-precision))) >> (latest_one-precision); - //unsigned shifted_output = num >> (latest_one-precision+1); - // if shifted_output is a number like 1111, don't round up - //if (shifted_output == (pow(2,precision)-1)) round_up = 0; - //num = shifted_output + round_up; - num >>= (latest_one-precision+1); - } - return num; -} -void mapping(int thread,int wmma_type,int wmma_layout,int type,int index,int stride,int &row,int &col,int &assg_offset){ - int offset; - int c_row_offset[]={0,8,0,8,4,12,4,12}; - int c_col_offset[]={0,0,8,8,0,0,8,8}; - int c_tg_inside_row_offset[]={0,1,0,1}; - int c_tg_inside_col_offset[]={0,0,2,2}; - int c_inside_row_offset[]={0,0,2,2,0,0,2,2}; - int c_inside_col_offset[]={0,1,0,1,4,5,4,5}; - - offset=thread_group_offset(thread,wmma_type,wmma_layout,type,stride); - - if(wmma_type==LOAD_A){ - if(wmma_layout==ROW){ - offset+=index+8*((thread%16)/8); - } - else{ - offset+=64*(index/4)+index%4+128*((thread%16)/8); - } - offset=(offset/16)*stride+offset%16; - assg_offset=index+8*((thread%16)/8); - } - else if(wmma_type==LOAD_B){ - if(wmma_layout==ROW){ - offset+=64*(index/4)+index%4+128*((thread%16)/8); - } - else{ - offset+=index+8*((thread%16)/8); - } - offset=(offset/16)*stride+offset%16; - assg_offset=index+8*((thread%16)/8); - } - else if( wmma_type==LOAD_C){ - if(type==F16_TYPE){ - row=c_row_offset[thread/4]+thread%4; - col=c_col_offset[thread/4]+index; - } - else{ - row=c_row_offset[thread/4]+c_tg_inside_row_offset[thread%4]+c_inside_row_offset[index]; - col=c_col_offset[thread/4]+c_tg_inside_col_offset[thread%4]+c_inside_col_offset[index]; - } - assg_offset=index; - } - - if(wmma_type==LOAD_A||wmma_type==LOAD_B){ - if(wmma_layout==ROW){ - row=offset/16; - col=offset%16; - } - else{ - col=offset/16; - row=offset%16; - } - } -} - -void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) -{ - int i,j,k,thrd; - int row,col,offset; - ptx_reg_t matrix_a[16][16]; - ptx_reg_t matrix_b[16][16]; - ptx_reg_t matrix_c[16][16]; - ptx_reg_t matrix_d[16][16]; - ptx_reg_t src_data; - ptx_thread_info *thread; - - unsigned a_layout = pI->get_wmma_layout(0); - unsigned b_layout = pI->get_wmma_layout(1); - unsigned type = pI->get_type(); - unsigned type2 = pI->get_type2(); - int tid ; - const operand_info &dst = pI->operand_lookup(0); - - if(core->get_gpu()->is_functional_sim()) - tid= inst.warp_id_func()*core->get_warp_size(); - else - tid= inst.warp_id()*core->get_warp_size(); - float temp; - half temp2; - - for (thrd=0; thrd < core->get_warp_size(); thrd++){ - thread = core->get_thread_info()[tid+thrd]; - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("THREAD=%d\n:",thrd); - for(int operand_num=1;operand_num<=3;operand_num++){ - const operand_info &src_a= pI->operand_lookup(operand_num); - unsigned nelem = src_a.get_vect_nelem(); - ptx_reg_t v[8]; - thread->get_vector_operand_values( src_a, v, nelem ); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("Thread%d_Iteration=%d\n:", thrd, operand_num); - for(k = 0; k < nelem; k++){ - printf("%llx ",v[k].u64); - } - printf("\n"); - } - ptx_reg_t nw_v[16]; - int hex_val; - - if(!((operand_num==3)&&(type2==F32_TYPE))){ - for(k=0;k<2*nelem;k++){ - if(k%2==1) - hex_val=(v[k/2].s64&0xffff); - else - hex_val=((v[k/2].s64&0xffff0000)>>16); - nw_v[k].f16 =*((half *)&hex_val); - } - } - if(!((operand_num==3)&&(type2==F32_TYPE))){ - for(k=0;k<2*nelem;k++){ - temp=nw_v[k].f16; - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("%.2f ",temp); - } - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("\n"); - } - else{ - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - for(k=0;k<8;k++){ - printf("%.2f ",v[k].f32); - } - printf("\n"); - } - } - switch(operand_num) { - case 1 ://operand 1 - for(k=0;k<8;k++){ - mapping(thrd,LOAD_A,a_layout,F16_TYPE,k,16,row,col,offset); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("A:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); - matrix_a[row][col]=nw_v[offset]; - } - break; - case 2 ://operand 2 - for(k=0;k<8;k++){ - mapping(thrd,LOAD_B,b_layout,F16_TYPE,k,16,row,col,offset); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("B:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); - matrix_b[row][col]=nw_v[offset]; - } - break; - case 3 ://operand 3 - for(k=0;k<8;k++){ - mapping(thrd,LOAD_C,ROW,type2,k,16,row,col,offset); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("C:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); - if(type2!=F16_TYPE){ - matrix_c[row][col]=v[offset]; - } - else { - matrix_c[row][col]=nw_v[offset]; - } - } - break; - default : - printf("Invalid Operand Index\n" ); - } - } - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("\n"); - } - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("MATRIX_A\n"); - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - temp=matrix_a[i][j].f16; - printf("%.2f ",temp); - } - printf("\n"); - } - printf("MATRIX_B\n"); - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - temp=matrix_b[i][j].f16; - printf("%.2f ",temp); - } - printf("\n"); - } - printf("MATRIX_C\n"); - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - if(type2==F16_TYPE){ - temp=matrix_c[i][j].f16; - printf("%.2f ",temp); - } - else - printf("%.2f ",matrix_c[i][j].f32); - } - printf("\n"); - } - } - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - matrix_d[i][j].f16=0; - } - } - - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - for(k=0;k<16;k++){ - matrix_d[i][j].f16=matrix_d[i][j].f16+matrix_a[i][k].f16*matrix_b[k][j].f16; - } - if((type==F16_TYPE)&&(type2==F16_TYPE)) - matrix_d[i][j].f16+=matrix_c[i][j].f16; - else if((type==F32_TYPE)&&(type2==F16_TYPE)){ - temp2=matrix_d[i][j].f16+matrix_c[i][j].f16; - temp=temp2; - matrix_d[i][j].f32=temp; - } - else if((type==F16_TYPE)&&(type2==F32_TYPE)){ - temp=matrix_d[i][j].f16; - temp+=matrix_c[i][j].f32; - matrix_d[i][j].f16=half(temp); - } - else{ - temp=matrix_d[i][j].f16; - temp+=matrix_c[i][j].f32; - matrix_d[i][j].f32=temp; - } - } - } - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("MATRIX_D\n"); - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - if(type==F16_TYPE){ - temp=matrix_d[i][j].f16; - printf("%.2f ",temp); - } - else - printf("%.2f ",matrix_d[i][j].f32); - } - printf("\n"); - } - } - for (thrd=0; thrd < core->get_warp_size(); thrd++){ - int row_t[8]; - int col_t[8]; - for(k=0;k<8;k++){ - mapping(thrd,LOAD_C,ROW,type,k,16,row_t[k],col_t[k],offset); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma:store:row:%d,col%d\n",row_t[k],col_t[k]); - } - thread = core->get_thread_info()[tid+thrd]; - - - if(type==F32_TYPE){ - thread->set_wmma_vector_operand_values(dst,matrix_d[row_t[0]][col_t[0]],matrix_d[row_t[1]][col_t[1]],matrix_d[row_t[2]][col_t[2]],matrix_d[row_t[3]][col_t[3]],matrix_d[row_t[4]][col_t[4]],matrix_d[row_t[5]][col_t[5]],matrix_d[row_t[6]][col_t[6]],matrix_d[row_t[7]][col_t[7]]); - - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - { - printf("thread%d:",thrd); - for(k=0;k<8;k++){ - printf("%.2f ",matrix_d[row_t[k]][col_t[k]].f32); - } - printf("\n"); - } - } - else if(type==F16_TYPE){ - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("thread%d:",thrd); - for(k=0;k<8;k++){ - temp=matrix_d[row_t[k]][col_t[k]].f16; - printf("%.2f ",temp); - } - printf("\n"); - - printf("thread%d:",thrd); - for(k=0;k<8;k++){ - printf("%x ", (unsigned int)matrix_d[row_t[k]][col_t[k]].f16); - } - printf("\n"); - } - ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4; - nw_data1.s64=((matrix_d[row_t[0]][col_t[0]].s64 & 0xffff))|((matrix_d[row_t[1]][col_t[1]].s64&0xffff)<<16); - nw_data2.s64=((matrix_d[row_t[2]][col_t[2]].s64 & 0xffff))|((matrix_d[row_t[3]][col_t[3]].s64&0xffff)<<16); - nw_data3.s64=((matrix_d[row_t[4]][col_t[4]].s64 & 0xffff))|((matrix_d[row_t[5]][col_t[5]].s64&0xffff)<<16); - nw_data4.s64=((matrix_d[row_t[6]][col_t[6]].s64 & 0xffff))|((matrix_d[row_t[7]][col_t[7]].s64&0xffff)<<16); - thread->set_vector_operand_values(dst,nw_data1,nw_data2,nw_data3,nw_data4); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("thread%d=%llx,%llx,%llx,%llx", thrd, nw_data1.s64, nw_data2.s64, nw_data3.s64, nw_data4.s64); - - } - else{ - printf("wmma:mma:wrong type\n"); - abort(); - } - } -} - -void call_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - static unsigned call_uid_next = 1; - - const operand_info &target = pI->func_addr(); - assert( target.is_function_address() ); - const symbol *func_addr = target.get_symbol(); - function_info *target_func = func_addr->get_pc(); - if (target_func->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", target_func->get_name().c_str() ); - } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", target_func->get_name().c_str() ); - /* - * Some of the instructions like printf() gives the gpgpusim the wrong impression that it is a function call. - * As printf() doesnt have a body like functions do, doing pdom analysis for printf() causes a crash. - */ - if (target_func->get_function_size() >0) - target_func->do_pdom(); - target_func->set_pdom(); - } - - // check that number of args and return match function requirements - if( pI->has_return() ^ target_func->has_return() ) { - printf("GPGPU-Sim PTX: Execution error - mismatch in number of return values between\n" - " call instruction and function declaration\n"); - abort(); - } - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - unsigned n_operands = pI->get_num_operands(); - - if( n_operands != (n_return+1+n_args) ) { - printf("GPGPU-Sim PTX: Execution error - mismatch in number of arguements between\n" - " call instruction and function declaration\n"); - abort(); - } - - // handle intrinsic functions - std::string fname = target_func->get_name(); - if( fname == "vprintf" ) { - gpgpusim_cuda_vprintf(pI, thread, target_func); - return; - } +void call_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + static unsigned call_uid_next = 1; + + const operand_info &target = pI->func_addr(); + assert(target.is_function_address()); + const symbol *func_addr = target.get_symbol(); + function_info *target_func = func_addr->get_pc(); + if (target_func->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", + target_func->get_name().c_str()); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", + target_func->get_name().c_str()); + /* + * Some of the instructions like printf() gives the gpgpusim the wrong + * impression that it is a function call. + * As printf() doesnt have a body like functions do, doing pdom analysis for + * printf() causes a crash. + */ + if (target_func->get_function_size() > 0) target_func->do_pdom(); + target_func->set_pdom(); + } + + // check that number of args and return match function requirements + if (pI->has_return() ^ target_func->has_return()) { + printf( + "GPGPU-Sim PTX: Execution error - mismatch in number of return values " + "between\n" + " call instruction and function declaration\n"); + abort(); + } + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + unsigned n_operands = pI->get_num_operands(); + + if (n_operands != (n_return + 1 + n_args)) { + printf( + "GPGPU-Sim PTX: Execution error - mismatch in number of arguements " + "between\n" + " call instruction and function declaration\n"); + abort(); + } + // handle intrinsic functions + std::string fname = target_func->get_name(); + if (fname == "vprintf") { + gpgpusim_cuda_vprintf(pI, thread, target_func); + return; + } #if (CUDART_VERSION >= 5000) - //Jin: handle device runtime apis for CDP - else if(fname == "cudaGetParameterBufferV2") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_getParameterBufferV2(pI, thread, target_func); - return; - } - else if(fname == "cudaLaunchDeviceV2") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_launchDeviceV2(pI, thread, target_func); - return; - } - else if(fname == "cudaStreamCreateWithFlags") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_streamCreateWithFlags(pI, thread, target_func); - return; - } + // Jin: handle device runtime apis for CDP + else if (fname == "cudaGetParameterBufferV2") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_getParameterBufferV2( + pI, thread, target_func); + return; + } else if (fname == "cudaLaunchDeviceV2") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_launchDeviceV2( + pI, thread, target_func); + return; + } else if (fname == "cudaStreamCreateWithFlags") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_streamCreateWithFlags( + pI, thread, target_func); + return; + } #endif - // read source arguements into register specified in declaration of function - arg_buffer_list_t arg_values; - copy_args_into_buffer_list(pI, thread, target_func, arg_values); + // read source arguements into register specified in declaration of function + arg_buffer_list_t arg_values; + copy_args_into_buffer_list(pI, thread, target_func, arg_values); - // record local for return value (we only support a single return value) - const symbol *return_var_src = NULL; - const symbol *return_var_dst = NULL; - if( target_func->has_return() ) { - return_var_dst = pI->dst().get_symbol(); - return_var_src = target_func->get_return_var(); - } + // record local for return value (we only support a single return value) + const symbol *return_var_src = NULL; + const symbol *return_var_dst = NULL; + if (target_func->has_return()) { + return_var_dst = pI->dst().get_symbol(); + return_var_src = target_func->get_return_var(); + } - gpgpu_sim *gpu = thread->get_gpu(); - unsigned callee_pc=0, callee_rpc=0; - if( gpu->simd_model() == POST_DOMINATOR ) { - thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(),&callee_pc,&callee_rpc); - assert( callee_pc == thread->get_pc() ); - } + gpgpu_sim *gpu = thread->get_gpu(); + unsigned callee_pc = 0, callee_rpc = 0; + if (gpu->simd_model() == POST_DOMINATOR) { + thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(), + &callee_pc, &callee_rpc); + assert(callee_pc == thread->get_pc()); + } - thread->callstack_push(callee_pc + pI->inst_size(), callee_rpc, return_var_src, return_var_dst, call_uid_next++); + thread->callstack_push(callee_pc + pI->inst_size(), callee_rpc, + return_var_src, return_var_dst, call_uid_next++); - copy_buffer_list_into_frame(thread, arg_values); + copy_buffer_list_into_frame(thread, arg_values); - thread->set_npc(target_func); + thread->set_npc(target_func); } -//Ptxplus version of call instruction. Jumps to a label not a different Kernel. -void callp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - - static unsigned call_uid_next = 1; +// Ptxplus version of call instruction. Jumps to a label not a different Kernel. +void callp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + static unsigned call_uid_next = 1; - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = + thread->get_operand_value(target, target, U32_TYPE, thread, 1); - const symbol *return_var_src = NULL; - const symbol *return_var_dst = NULL; + const symbol *return_var_src = NULL; + const symbol *return_var_dst = NULL; - gpgpu_sim *gpu = thread->get_gpu(); - unsigned callee_pc=0, callee_rpc=0; - if( gpu->simd_model() == POST_DOMINATOR ) { - thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(),&callee_pc,&callee_rpc); - assert( callee_pc == thread->get_pc() ); - } + gpgpu_sim *gpu = thread->get_gpu(); + unsigned callee_pc = 0, callee_rpc = 0; + if (gpu->simd_model() == POST_DOMINATOR) { + thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(), + &callee_pc, &callee_rpc); + assert(callee_pc == thread->get_pc()); + } - thread->callstack_push_plus(callee_pc + pI->inst_size(), callee_rpc, return_var_src, return_var_dst, call_uid_next++); - thread->set_npc(target_pc); + thread->callstack_push_plus(callee_pc + pI->inst_size(), callee_rpc, + return_var_src, return_var_dst, call_uid_next++); + thread->set_npc(target_pc); } -void clz_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void clz_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - int max; - unsigned long long mask; - d.u64 = 0; + int max; + unsigned long long mask; + d.u64 = 0; - switch ( i_type ) { - case B32_TYPE: + switch (i_type) { + case B32_TYPE: max = 32; mask = 0x80000000; break; - case B64_TYPE: + case B64_TYPE: max = 64; mask = 0x8000000000000000; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - while ((d.u32 < max) && ((a.u64&mask) == 0) ) { - d.u32++; - a.u64 = a.u64 << 1; - } + while ((d.u32 < max) && ((a.u64 & mask) == 0)) { + d.u32++; + a.u64 = a.u64 << 1; + } - thread->set_operand_value(dst,d, B32_TYPE, thread, pI); + thread->set_operand_value(dst, d, B32_TYPE, thread, pI); } -void cnot_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void cnot_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch ( i_type ) { - case PRED_TYPE: d.pred = ((a.pred & 0x0001) == 0)?1:0; break; - case B16_TYPE: d.u16 = (a.u16 == 0)?1:0; break; - case B32_TYPE: d.u32 = (a.u32 == 0)?1:0; break; - case B64_TYPE: d.u64 = (a.u64 == 0)?1:0; break; - default: + switch (i_type) { + case PRED_TYPE: + d.pred = ((a.pred & 0x0001) == 0) ? 1 : 0; + break; + case B16_TYPE: + d.u16 = (a.u16 == 0) ? 1 : 0; + break; + case B32_TYPE: + d.u32 = (a.u32 == 0) ? 1 : 0; + break; + case B64_TYPE: + d.u64 = (a.u64 == 0) ? 1 : 0; + break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void cos_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void cos_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - switch ( i_type ) { - case F32_TYPE: + switch (i_type) { + case F32_TYPE: d.f32 = cos(a.f32); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - - thread->set_operand_value(dst,d, i_type, thread, pI); -} - -ptx_reg_t chop( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - switch ( to_width ) { - case 8: x.mask_and(0,0xFF); break; - case 16: x.mask_and(0,0xFFFF); break; - case 32: x.mask_and(0,0xFFFFFFFF); break; - case 64: break; - default: assert(0); - } - return x; -} - -ptx_reg_t sext( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - x=chop(x,0,from_width,0,rounding_mode,saturation_mode); - switch ( from_width ) { - case 8: if ( x.get_bit(7) ) x.mask_or(0xFFFFFFFF,0xFFFFFF00);break; - case 16:if ( x.get_bit(15) ) x.mask_or(0xFFFFFFFF,0xFFFF0000);break; - case 32: if ( x.get_bit(31) ) x.mask_or(0xFFFFFFFF,0x00000000);break; - case 64: break; - default: assert(0); - } - return x; -} - -// sign extend depending on the destination register size - hack to get SobelFilter working in CUDA 4.2 -ptx_reg_t sexd( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - x=chop(x,0,from_width,0,rounding_mode,saturation_mode); - switch ( to_width ) { - case 8: if ( x.get_bit(7) ) x.mask_or(0xFFFFFFFF,0xFFFFFF00);break; - case 16:if ( x.get_bit(15) ) x.mask_or(0xFFFFFFFF,0xFFFF0000);break; - case 32: if ( x.get_bit(31) ) x.mask_or(0xFFFFFFFF,0x00000000);break; - case 64: break; - default: assert(0); - } - return x; -} - -ptx_reg_t zext( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - return chop(x,0,from_width,0,rounding_mode,saturation_mode); -} - -int saturatei(int a, int max, int min) -{ - if (a > max) a = max; - else if (a < min) a = min; - return a; -} - -unsigned int saturatei(unsigned int a, unsigned int max) -{ - if (a > max) a = max; - return a; -} - -ptx_reg_t f2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - half mytemp; - half_float::half tmp_h; - //assert( from_width == 32); - - enum cudaRoundMode mode = cudaRoundZero; - switch (rounding_mode) { - case RZI_OPTION: mode = cudaRoundZero; break; - case RNI_OPTION: mode = cudaRoundNearest; break; - case RMI_OPTION: mode = cudaRoundMinInf; break; - case RPI_OPTION: mode = cudaRoundPosInf; break; - default: break; - } - - ptx_reg_t y; - if ( to_sign == 1 ) { // convert to 64-bit number first? - int tmp = cuda_math::float2int(x.f32, mode); - if ((x.u32 & 0x7f800000) == 0) - tmp = 0; // round denorm. FP to 0 - if (saturation_mode && to_width < 32) { - tmp = saturatei(tmp, (1<::round_style>(x.f32);//mytemp; - break; - case 32: - y.f32=float(x.f16); - break; // handled by f2f - case 64: - y.f64 = x.f32; - break; - default: assert(0); break; - } - } - return y; -} - -double saturated2i (double a, double max, double min) { - if (a > max) a = max; - else if (a < min) a = min; - return a; -} - -ptx_reg_t d2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - assert( from_width == 64); - - double tmp; - switch (rounding_mode) { - case RZI_OPTION: tmp = trunc(x.f64); break; - case RNI_OPTION: tmp = nearbyint(x.f64); break; - case RMI_OPTION: tmp = floor(x.f64); break; - case RPI_OPTION: tmp = ceil(x.f64); break; - default: tmp = x.f64; break; - } - - ptx_reg_t y; - if ( to_sign == 1 ) { - tmp = saturated2i(tmp, ((1<<(to_width - 1)) - 1), (1<<(to_width - 1)) ); - switch ( to_width ) { - case 8: y.s8 = (char)tmp; break; - case 16: y.s16 = (short)tmp; break; - case 32: y.s32 = (int)tmp; break; - case 64: y.s64 = (long long)tmp; break; - default: assert(0); break; - } - } else if ( to_sign == 0 ) { - tmp = saturated2i(tmp, ((1<<(to_width - 1)) - 1), 0); - switch ( to_width ) { - case 8: y.u8 = (unsigned char)tmp; break; - case 16: y.u16 = (unsigned short)tmp; break; - case 32: y.u32 = (unsigned int)tmp; break; - case 64: y.u64 = (unsigned long long)tmp; break; - default: assert(0); break; - } - } else { - switch ( to_width ) { - case 16: assert(0); break; - case 32: - y.f32 = x.f64; - break; - case 64: - y.f64 = x.f64; // should be handled by d2d - break; - default: assert(0); break; - } - } - return y; -} - -ptx_reg_t s2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - ptx_reg_t y; - - if (from_width < 64) { // 32-bit conversion - y = sext(x,from_width,32,0,rounding_mode,saturation_mode); - - switch ( to_width ) { - case 16: assert(0); break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: y.f32 = cuda_math::__int2float_rz(y.s32); break; - case RN_OPTION: y.f32 = cuda_math::__int2float_rn(y.s32); break; - case RM_OPTION: y.f32 = cuda_math::__int2float_rd(y.s32); break; - case RP_OPTION: y.f32 = cuda_math::__int2float_ru(y.s32); break; - default: break; - } - break; - case 64: y.f64 = y.s32; break; // no rounding needed - default: assert(0); break; - } - } else { - switch ( to_width ) { - case 16: assert(0); break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: y.f32 = cuda_math::__ll2float_rz(y.s64); break; - case RN_OPTION: y.f32 = cuda_math::__ll2float_rn(y.s64); break; - case RM_OPTION: y.f32 = cuda_math::__ll2float_rd(y.s64); break; - case RP_OPTION: y.f32 = cuda_math::__ll2float_ru(y.s64); break; - default: break; - } - break; - case 64: y.f64 = y.s64; break; // no internal implementation found - default: assert(0); break; - } - } - - // saturating an integer to 1 or 0? - return y; -} - -ptx_reg_t u2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - ptx_reg_t y; - - if (from_width < 64) { // 32-bit conversion - y = zext(x,from_width,32,0,rounding_mode,saturation_mode); - - switch ( to_width ) { - case 16: assert(0); break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: y.f32 = cuda_math::__uint2float_rz(y.u32); break; - case RN_OPTION: y.f32 = cuda_math::__uint2float_rn(y.u32); break; - case RM_OPTION: y.f32 = cuda_math::__uint2float_rd(y.u32); break; - case RP_OPTION: y.f32 = cuda_math::__uint2float_ru(y.u32); break; - default: break; - } - break; - case 64: y.f64 = y.u32; break; // no rounding needed - default: assert(0); break; - } - } else { - switch ( to_width ) { - case 16: assert(0); break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; - case RN_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; - case RM_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; - case RP_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; - default: break; - } - break; - case 64: y.f64 = y.u64; break; // no internal implementation found - default: assert(0); break; - } - } - - // saturating an integer to 1 or 0? - return y; -} - -ptx_reg_t f2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - ptx_reg_t y; - if (from_width == 16){ - half_float::detail::uint16 val = x.u16; - y.f32 = half_float::detail::half2float(val); - }else{ - switch ( rounding_mode ) { - case RZI_OPTION: - y.f32 = truncf(x.f32); - break; - case RNI_OPTION: - #if CUDART_VERSION >= 3000 - y.f32 = nearbyintf(x.f32); - #else - y.f32 = cuda_math::__internal_nearbyintf(x.f32); - #endif - break; - case RMI_OPTION: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = floorf(x.f32); - } - break; - case RPI_OPTION: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = ceilf(x.f32); - } - break; - default: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = x.f32; - } - break; - } - #if CUDART_VERSION >= 3000 - if (isnanf(y.f32)) - #else - if (cuda_math::__cuda___isnanf(y.f32)) - #endif - { - y.u32 = 0x7fffffff; - } else if (saturation_mode) { - y.f32 = cuda_math::__saturatef(y.f32); - } - } - - return y; -} - -ptx_reg_t d2d( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - ptx_reg_t y; - switch ( rounding_mode ) { - case RZI_OPTION: - y.f64 = trunc(x.f64); - break; - case RNI_OPTION: -#if CUDART_VERSION >= 3000 - y.f64 = nearbyint(x.f64); -#else - y.f64 = cuda_math::__internal_nearbyintf(x.f64); -#endif - break; - case RMI_OPTION: - y.f64 = floor(x.f64); - break; - case RPI_OPTION: - y.f64 = ceil(x.f64); - break; - default: - y.f64 = x.f64; - break; - } - if (std::isnan(y.f64)) { - y.u64 = 0xfff8000000000000ull; - } else if (saturation_mode) { - y.f64 = cuda_math::__saturatef(y.f64); - } - return y; -} - -ptx_reg_t (*g_cvt_fn[11][11])( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode ) = { - { NULL, sext, sext, sext, NULL, sext, sext, sext, s2f, s2f, s2f}, - { chop, NULL, sext, sext, chop, NULL, sext, sext, s2f, s2f, s2f}, - { chop, sexd, NULL, sext, chop, chop, NULL, sext, s2f, s2f, s2f}, - { chop, chop, chop, NULL, chop, chop, chop, NULL, s2f, s2f, s2f}, - { NULL, zext, zext, zext, NULL, zext, zext, zext, u2f, u2f, u2f}, - { chop, NULL, zext, zext, chop, NULL, zext, zext, u2f, u2f, u2f}, - { chop, chop, NULL, zext, chop, chop, NULL, zext, u2f, u2f, u2f}, - { chop, chop, chop, NULL, chop, chop, chop, NULL, u2f, u2f, u2f}, - { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , NULL,f2f, f2x}, - { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x, f2f, f2x}, - { d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x, d2x, d2d} -}; - -void ptx_round(ptx_reg_t& data, int rounding_mode, int type) -{ - if (rounding_mode == RN_OPTION) { - return; - } - switch ( rounding_mode ) { - case RZI_OPTION: - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: data.f16=truncf(data.f16);break;//assert(0); break; - case F32_TYPE: - data.f32 = truncf(data.f32); - break; - case F64_TYPE: - case FF64_TYPE: - if (data.f64 < 0) data.f64 = ceil(data.f64); //negative - else data.f64 = floor(data.f64); //positive - break; - default: assert(0); break; - } + assert(0); break; - case RNI_OPTION: - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE:// assert(0); break; + } + + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +ptx_reg_t chop(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + switch (to_width) { + case 8: + x.mask_and(0, 0xFF); + break; + case 16: + x.mask_and(0, 0xFFFF); + break; + case 32: + x.mask_and(0, 0xFFFFFFFF); + break; + case 64: + break; + default: + assert(0); + } + return x; +} + +ptx_reg_t sext(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + x = chop(x, 0, from_width, 0, rounding_mode, saturation_mode); + switch (from_width) { + case 8: + if (x.get_bit(7)) x.mask_or(0xFFFFFFFF, 0xFFFFFF00); + break; + case 16: + if (x.get_bit(15)) x.mask_or(0xFFFFFFFF, 0xFFFF0000); + break; + case 32: + if (x.get_bit(31)) x.mask_or(0xFFFFFFFF, 0x00000000); + break; + case 64: + break; + default: + assert(0); + } + return x; +} + +// sign extend depending on the destination register size - hack to get +// SobelFilter working in CUDA 4.2 +ptx_reg_t sexd(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + x = chop(x, 0, from_width, 0, rounding_mode, saturation_mode); + switch (to_width) { + case 8: + if (x.get_bit(7)) x.mask_or(0xFFFFFFFF, 0xFFFFFF00); + break; + case 16: + if (x.get_bit(15)) x.mask_or(0xFFFFFFFF, 0xFFFF0000); + break; + case 32: + if (x.get_bit(31)) x.mask_or(0xFFFFFFFF, 0x00000000); + break; + case 64: + break; + default: + assert(0); + } + return x; +} + +ptx_reg_t zext(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + return chop(x, 0, from_width, 0, rounding_mode, saturation_mode); +} + +int saturatei(int a, int max, int min) { + if (a > max) + a = max; + else if (a < min) + a = min; + return a; +} + +unsigned int saturatei(unsigned int a, unsigned int max) { + if (a > max) a = max; + return a; +} + +ptx_reg_t f2x(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + half mytemp; + half_float::half tmp_h; + // assert( from_width == 32); + + enum cudaRoundMode mode = cudaRoundZero; + switch (rounding_mode) { + case RZI_OPTION: + mode = cudaRoundZero; + break; + case RNI_OPTION: + mode = cudaRoundNearest; + break; + case RMI_OPTION: + mode = cudaRoundMinInf; + break; + case RPI_OPTION: + mode = cudaRoundPosInf; + break; + default: + break; + } + + ptx_reg_t y; + if (to_sign == 1) { // convert to 64-bit number first? + int tmp = cuda_math::float2int(x.f32, mode); + if ((x.u32 & 0x7f800000) == 0) tmp = 0; // round denorm. FP to 0 + if (saturation_mode && to_width < 32) { + tmp = saturatei(tmp, (1 << to_width) - 1, -(1 << to_width)); + } + switch (to_width) { + case 8: + y.s8 = (char)tmp; + break; + case 16: + y.s16 = (short)tmp; + break; + case 32: + y.s32 = (int)tmp; + break; + case 64: + y.s64 = (long long)tmp; + break; + default: + assert(0); + break; + } + } else if (to_sign == 0) { + unsigned int tmp = cuda_math::float2uint(x.f32, mode); + if ((x.u32 & 0x7f800000) == 0) tmp = 0; // round denorm. FP to 0 + if (saturation_mode && to_width < 32) { + tmp = saturatei(tmp, (1 << to_width) - 1); + } + switch (to_width) { + case 8: + y.u8 = (unsigned char)tmp; + break; + case 16: + y.u16 = (unsigned short)tmp; + break; + case 32: + y.u32 = (unsigned int)tmp; + break; + case 64: + y.u64 = (unsigned long long)tmp; + break; + default: + assert(0); + break; + } + } else { + switch (to_width) { + case 16: + y.f16 = half_float::half_cast::round_style>( + x.f32); // mytemp; + break; + case 32: + y.f32 = float(x.f16); + break; // handled by f2f + case 64: + y.f64 = x.f32; + break; + default: + assert(0); + break; + } + } + return y; +} + +double saturated2i(double a, double max, double min) { + if (a > max) + a = max; + else if (a < min) + a = min; + return a; +} + +ptx_reg_t d2x(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + assert(from_width == 64); + + double tmp; + switch (rounding_mode) { + case RZI_OPTION: + tmp = trunc(x.f64); + break; + case RNI_OPTION: + tmp = nearbyint(x.f64); + break; + case RMI_OPTION: + tmp = floor(x.f64); + break; + case RPI_OPTION: + tmp = ceil(x.f64); + break; + default: + tmp = x.f64; + break; + } + + ptx_reg_t y; + if (to_sign == 1) { + tmp = saturated2i(tmp, ((1 << (to_width - 1)) - 1), (1 << (to_width - 1))); + switch (to_width) { + case 8: + y.s8 = (char)tmp; + break; + case 16: + y.s16 = (short)tmp; + break; + case 32: + y.s32 = (int)tmp; + break; + case 64: + y.s64 = (long long)tmp; + break; + default: + assert(0); + break; + } + } else if (to_sign == 0) { + tmp = saturated2i(tmp, ((1 << (to_width - 1)) - 1), 0); + switch (to_width) { + case 8: + y.u8 = (unsigned char)tmp; + break; + case 16: + y.u16 = (unsigned short)tmp; + break; + case 32: + y.u32 = (unsigned int)tmp; + break; + case 64: + y.u64 = (unsigned long long)tmp; + break; + default: + assert(0); + break; + } + } else { + switch (to_width) { + case 16: + assert(0); + break; + case 32: + y.f32 = x.f64; + break; + case 64: + y.f64 = x.f64; // should be handled by d2d + break; + default: + assert(0); + break; + } + } + return y; +} + +ptx_reg_t s2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + ptx_reg_t y; + + if (from_width < 64) { // 32-bit conversion + y = sext(x, from_width, 32, 0, rounding_mode, saturation_mode); + + switch (to_width) { + case 16: + assert(0); + break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: + y.f32 = cuda_math::__int2float_rz(y.s32); + break; + case RN_OPTION: + y.f32 = cuda_math::__int2float_rn(y.s32); + break; + case RM_OPTION: + y.f32 = cuda_math::__int2float_rd(y.s32); + break; + case RP_OPTION: + y.f32 = cuda_math::__int2float_ru(y.s32); + break; + default: + break; + } + break; + case 64: + y.f64 = y.s32; + break; // no rounding needed + default: + assert(0); + break; + } + } else { + switch (to_width) { + case 16: + assert(0); + break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: + y.f32 = cuda_math::__ll2float_rz(y.s64); + break; + case RN_OPTION: + y.f32 = cuda_math::__ll2float_rn(y.s64); + break; + case RM_OPTION: + y.f32 = cuda_math::__ll2float_rd(y.s64); + break; + case RP_OPTION: + y.f32 = cuda_math::__ll2float_ru(y.s64); + break; + default: + break; + } + break; + case 64: + y.f64 = y.s64; + break; // no internal implementation found + default: + assert(0); + break; + } + } + + // saturating an integer to 1 or 0? + return y; +} + +ptx_reg_t u2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + ptx_reg_t y; + + if (from_width < 64) { // 32-bit conversion + y = zext(x, from_width, 32, 0, rounding_mode, saturation_mode); + + switch (to_width) { + case 16: + assert(0); + break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: + y.f32 = cuda_math::__uint2float_rz(y.u32); + break; + case RN_OPTION: + y.f32 = cuda_math::__uint2float_rn(y.u32); + break; + case RM_OPTION: + y.f32 = cuda_math::__uint2float_rd(y.u32); + break; + case RP_OPTION: + y.f32 = cuda_math::__uint2float_ru(y.u32); + break; + default: + break; + } + break; + case 64: + y.f64 = y.u32; + break; // no rounding needed + default: + assert(0); + break; + } + } else { + switch (to_width) { + case 16: + assert(0); + break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: + y.f32 = cuda_math::__ull2float_rn(y.u64); + break; + case RN_OPTION: + y.f32 = cuda_math::__ull2float_rn(y.u64); + break; + case RM_OPTION: + y.f32 = cuda_math::__ull2float_rn(y.u64); + break; + case RP_OPTION: + y.f32 = cuda_math::__ull2float_rn(y.u64); + break; + default: + break; + } + break; + case 64: + y.f64 = y.u64; + break; // no internal implementation found + default: + assert(0); + break; + } + } + + // saturating an integer to 1 or 0? + return y; +} + +ptx_reg_t f2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + ptx_reg_t y; + if (from_width == 16) { + half_float::detail::uint16 val = x.u16; + y.f32 = half_float::detail::half2float(val); + } else { + switch (rounding_mode) { + case RZI_OPTION: + y.f32 = truncf(x.f32); + break; + case RNI_OPTION: #if CUDART_VERSION >= 3000 - data.f16 = nearbyintf(data.f16); + y.f32 = nearbyintf(x.f32); #else - data.f16 = cuda_math::__cuda_nearbyintf(data.f16); + y.f32 = cuda_math::__internal_nearbyintf(x.f32); #endif - break; - case F32_TYPE: + break; + case RMI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = floorf(x.f32); + } + break; + case RPI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = ceilf(x.f32); + } + break; + default: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = x.f32; + } + break; + } #if CUDART_VERSION >= 3000 - data.f32 = nearbyintf(data.f32); + if (isnanf(y.f32)) #else - data.f32 = cuda_math::__cuda_nearbyintf(data.f32); + if (cuda_math::__cuda___isnanf(y.f32)) #endif - break; - case F64_TYPE: case FF64_TYPE: data.f64 = round(data.f64); break; - default: assert(0); break; + { + y.u32 = 0x7fffffff; + } else if (saturation_mode) { + y.f32 = cuda_math::__saturatef(y.f32); + } + } + + return y; +} + +ptx_reg_t d2d(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + ptx_reg_t y; + switch (rounding_mode) { + case RZI_OPTION: + y.f64 = trunc(x.f64); + break; + case RNI_OPTION: +#if CUDART_VERSION >= 3000 + y.f64 = nearbyint(x.f64); +#else + y.f64 = cuda_math::__internal_nearbyintf(x.f64); +#endif + break; + case RMI_OPTION: + y.f64 = floor(x.f64); + break; + case RPI_OPTION: + y.f64 = ceil(x.f64); + break; + default: + y.f64 = x.f64; + break; + } + if (std::isnan(y.f64)) { + y.u64 = 0xfff8000000000000ull; + } else if (saturation_mode) { + y.f64 = cuda_math::__saturatef(y.f64); + } + return y; +} + +ptx_reg_t (*g_cvt_fn[11][11])(ptx_reg_t x, unsigned from_width, + unsigned to_width, int to_sign, int rounding_mode, + int saturation_mode) = { + {NULL, sext, sext, sext, NULL, sext, sext, sext, s2f, s2f, s2f}, + {chop, NULL, sext, sext, chop, NULL, sext, sext, s2f, s2f, s2f}, + {chop, sexd, NULL, sext, chop, chop, NULL, sext, s2f, s2f, s2f}, + {chop, chop, chop, NULL, chop, chop, chop, NULL, s2f, s2f, s2f}, + {NULL, zext, zext, zext, NULL, zext, zext, zext, u2f, u2f, u2f}, + {chop, NULL, zext, zext, chop, NULL, zext, zext, u2f, u2f, u2f}, + {chop, chop, NULL, zext, chop, chop, NULL, zext, u2f, u2f, u2f}, + {chop, chop, chop, NULL, chop, chop, chop, NULL, u2f, u2f, u2f}, + {f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, NULL, f2f, f2x}, + {f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2f, f2x}, + {d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2d}}; + +void ptx_round(ptx_reg_t &data, int rounding_mode, int type) { + if (rounding_mode == RN_OPTION) { + return; + } + switch (rounding_mode) { + case RZI_OPTION: + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); + assert(0); + break; + case F16_TYPE: + data.f16 = truncf(data.f16); + break; // assert(0); break; + case F32_TYPE: + data.f32 = truncf(data.f32); + break; + case F64_TYPE: + case FF64_TYPE: + if (data.f64 < 0) + data.f64 = ceil(data.f64); // negative + else + data.f64 = floor(data.f64); // positive + break; + default: + assert(0); + break; } break; - case RMI_OPTION: - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: data.f16=floorf(data.f16);break;//assert(0); break; - case F32_TYPE: - data.f32 = floorf(data.f32); - break; - case F64_TYPE: case FF64_TYPE: data.f64 = floor(data.f64); break; - default: assert(0); break; + case RNI_OPTION: + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); + assert(0); + break; + case F16_TYPE: // assert(0); break; +#if CUDART_VERSION >= 3000 + data.f16 = nearbyintf(data.f16); +#else + data.f16 = cuda_math::__cuda_nearbyintf(data.f16); +#endif + break; + case F32_TYPE: +#if CUDART_VERSION >= 3000 + data.f32 = nearbyintf(data.f32); +#else + data.f32 = cuda_math::__cuda_nearbyintf(data.f32); +#endif + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = round(data.f64); + break; + default: + assert(0); + break; } break; - case RPI_OPTION: - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: data.f16 = ceilf(data.f16); break; //assert(0); break; - case F32_TYPE: data.f32 = ceilf(data.f32); break; - case F64_TYPE: case FF64_TYPE: data.f64 = ceil(data.f64); break; - default: assert(0); break; + case RMI_OPTION: + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); + assert(0); + break; + case F16_TYPE: + data.f16 = floorf(data.f16); + break; // assert(0); break; + case F32_TYPE: + data.f32 = floorf(data.f32); + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = floor(data.f64); + break; + default: + assert(0); + break; } break; - default: break; - } + case RPI_OPTION: + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); + assert(0); + break; + case F16_TYPE: + data.f16 = ceilf(data.f16); + break; // assert(0); break; + case F32_TYPE: + data.f32 = ceilf(data.f32); + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = ceil(data.f64); + break; + default: + assert(0); + break; + } + break; + default: + break; + } - if (type == F32_TYPE) { + if (type == F32_TYPE) { #if CUDART_VERSION >= 3000 - if (isnanf(data.f32)) + if (isnanf(data.f32)) #else - if (cuda_math::__cuda___isnanf(data.f32)) + if (cuda_math::__cuda___isnanf(data.f32)) #endif - { - data.u32 = 0x7fffffff; - } - } - if ((type == F64_TYPE)||(type == FF64_TYPE)) { - if (std::isnan(data.f64)) { - data.u64 = 0xfff8000000000000ull; - } - } + { + data.u32 = 0x7fffffff; + } + } + if ((type == F64_TYPE) || (type == FF64_TYPE)) { + if (std::isnan(data.f64)) { + data.u64 = 0xfff8000000000000ull; + } + } } -void ptx_saturate(ptx_reg_t& data, int saturation_mode, int type) -{ - if (!saturation_mode) { - return; - } - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to clamp an integer to 1??\n"); assert(0); break; - case F16_TYPE: //assert(0); break; - if (data.f16 > 1.0f) data.f16 = 1.0f; //negative - if (data.f16 < 0.0f) data.f16 = 0.0f; //positive - break; - case F32_TYPE: - if (data.f32 > 1.0f) data.f32 = 1.0f; //negative - if (data.f32 < 0.0f) data.f32 = 0.0f; //positive - break; - case F64_TYPE: - case FF64_TYPE: - if (data.f64 > 1.0f) data.f64 = 1.0f; //negative - if (data.f64 < 0.0f) data.f64 = 0.0f; //positive - break; - default: assert(0); break; - } - -} - -void cvt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned to_type = pI->get_type(); - unsigned from_type = pI->get_type2(); - unsigned rounding_mode = pI->rounding_mode(); - unsigned saturation_mode = pI->saturation_mode(); - -// if ( to_type == F16_TYPE || from_type == F16_TYPE ) -// abort(); - - int to_sign, from_sign; - size_t from_width, to_width; - unsigned src_fmt = type_info_key::type_decode(from_type, from_width, from_sign); - unsigned dst_fmt = type_info_key::type_decode(to_type, to_width, to_sign); - - ptx_reg_t data = thread->get_operand_value(src1, dst, from_type, thread, 1); - - if(pI->is_neg()){ - - switch( from_type ) { +void ptx_saturate(ptx_reg_t &data, int saturation_mode, int type) { + if (!saturation_mode) { + return; + } + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to clamp an integer to 1??\n"); + assert(0); + break; + case F16_TYPE: // assert(0); break; + if (data.f16 > 1.0f) data.f16 = 1.0f; // negative + if (data.f16 < 0.0f) data.f16 = 0.0f; // positive + break; + case F32_TYPE: + if (data.f32 > 1.0f) data.f32 = 1.0f; // negative + if (data.f32 < 0.0f) data.f32 = 0.0f; // positive + break; + case F64_TYPE: + case FF64_TYPE: + if (data.f64 > 1.0f) data.f64 = 1.0f; // negative + if (data.f64 < 0.0f) data.f64 = 0.0f; // positive + break; + default: + assert(0); + break; + } +} + +void cvt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned to_type = pI->get_type(); + unsigned from_type = pI->get_type2(); + unsigned rounding_mode = pI->rounding_mode(); + unsigned saturation_mode = pI->saturation_mode(); + + // if ( to_type == F16_TYPE || from_type == F16_TYPE ) + // abort(); + + int to_sign, from_sign; + size_t from_width, to_width; + unsigned src_fmt = + type_info_key::type_decode(from_type, from_width, from_sign); + unsigned dst_fmt = type_info_key::type_decode(to_type, to_width, to_sign); + + ptx_reg_t data = thread->get_operand_value(src1, dst, from_type, thread, 1); + + if (pI->is_neg()) { + switch (from_type) { // Default to f32 for now, need to add support for others case S8_TYPE: case U8_TYPE: case B8_TYPE: - data.s8 = -data.s8; - break; + data.s8 = -data.s8; + break; case S16_TYPE: case U16_TYPE: case B16_TYPE: - data.s16 = -data.s16; - break; + data.s16 = -data.s16; + break; case S32_TYPE: case U32_TYPE: case B32_TYPE: - data.s32 = -data.s32; - break; + data.s32 = -data.s32; + break; case S64_TYPE: case U64_TYPE: case B64_TYPE: - data.s64 = -data.s64; - break; + data.s64 = -data.s64; + break; case F16_TYPE: - data.f16 = -data.f16; - break; + data.f16 = -data.f16; + break; case F32_TYPE: - data.f32 = -data.f32; - break; + data.f32 = -data.f32; + break; case F64_TYPE: case FF64_TYPE: - data.f64 = -data.f64; - break; + data.f64 = -data.f64; + break; default: - assert(0); - } + assert(0); + } + } - } + if (g_cvt_fn[src_fmt][dst_fmt] != NULL) { + ptx_reg_t result = g_cvt_fn[src_fmt][dst_fmt]( + data, from_width, to_width, to_sign, rounding_mode, saturation_mode); + data = result; + } + thread->set_operand_value(dst, data, to_type, thread, pI); +} - if ( g_cvt_fn[src_fmt][dst_fmt] != NULL ) { - ptx_reg_t result = g_cvt_fn[src_fmt][dst_fmt](data,from_width,to_width,to_sign, rounding_mode, saturation_mode); - data = result; - } +void cvta_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + memory_space_t space = pI->get_space(); + bool to_non_generic = pI->is_to(); + + unsigned i_type = pI->get_type(); + ptx_reg_t from_addr = thread->get_operand_value(src1, dst, i_type, thread, 1); + addr_t from_addr_hw = (addr_t)from_addr.u64; + addr_t to_addr_hw = 0; + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); + + if (to_non_generic) { + switch (space.get_type()) { + case shared_space: + to_addr_hw = generic_to_shared(smid, from_addr_hw); + break; + case local_space: + to_addr_hw = generic_to_local(smid, hwtid, from_addr_hw); + break; + case global_space: + to_addr_hw = generic_to_global(from_addr_hw); + break; + default: + abort(); + } + } else { + switch (space.get_type()) { + case shared_space: + to_addr_hw = shared_to_generic(smid, from_addr_hw); + break; + case local_space: + to_addr_hw = local_to_generic(smid, hwtid, from_addr_hw) + + thread->get_local_mem_stack_pointer(); + break; // add stack ptr here so that it can be passed as a pointer at + // function call + case global_space: + to_addr_hw = global_to_generic(from_addr_hw); + break; + default: + abort(); + } + } - thread->set_operand_value(dst, data, to_type, thread, pI ); + ptx_reg_t to_addr; + to_addr.u64 = to_addr_hw; + thread->set_reg(dst.get_symbol(), to_addr); } -void cvta_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; +void div_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + switch (i_type) { + case S8_TYPE: + data.s8 = src1_data.s8 / src2_data.s8; + break; + case S16_TYPE: + data.s16 = src1_data.s16 / src2_data.s16; + break; + case S32_TYPE: + data.s32 = src1_data.s32 / src2_data.s32; + break; + case S64_TYPE: + data.s64 = src1_data.s64 / src2_data.s64; + break; + case U8_TYPE: + data.u8 = src1_data.u8 / src2_data.u8; + break; + case U16_TYPE: + data.u16 = src1_data.u16 / src2_data.u16; + break; + case U32_TYPE: + data.u32 = src1_data.u32 / src2_data.u32; + break; + case U64_TYPE: + data.u64 = src1_data.u64 / src2_data.u64; + break; + case B8_TYPE: + data.u8 = src1_data.u8 / src2_data.u8; + break; + case B16_TYPE: + data.u16 = src1_data.u16 / src2_data.u16; + break; + case B32_TYPE: + data.u32 = src1_data.u32 / src2_data.u32; + break; + case B64_TYPE: + data.u64 = src1_data.u64 / src2_data.u64; + break; + case F16_TYPE: + data.f16 = src1_data.f16 / src2_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = src1_data.f32 / src2_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = src1_data.f64 / src2_data.f64; + break; + default: + assert(0); + break; + } + thread->set_operand_value(dst, data, i_type, thread, pI); +} + +void dp4a_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + printf("DP4A instruction not implemented yet"); + assert(0); +} + +void ex2_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch (i_type) { + case F32_TYPE: + data.f32 = cuda_math::__powf(2.0, src1_data.f32); + break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst, data, i_type, thread, pI); +} + +void exit_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); +} + +void mad_def(const ptx_instruction *pI, ptx_thread_info *thread, + bool use_carry = false); + +void fma_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + mad_def(pI, thread); +} + +void isspacep_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a; + bool t = false; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + memory_space_t space = pI->get_space(); + + a = thread->get_reg(src1.get_symbol()); + addr_t addr = (addr_t)a.u64; + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); + + switch (space.get_type()) { + case shared_space: + t = isspace_shared(smid, addr); + case local_space: + t = isspace_local(smid, hwtid, addr); + case global_space: + t = isspace_global(addr); + default: + abort(); + } - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - memory_space_t space = pI->get_space(); - bool to_non_generic = pI->is_to(); + ptx_reg_t p; + p.pred = t ? 1 : 0; - unsigned i_type = pI->get_type(); - ptx_reg_t from_addr = thread->get_operand_value(src1,dst,i_type,thread,1); - addr_t from_addr_hw = (addr_t)from_addr.u64; - addr_t to_addr_hw = 0; - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); + thread->set_reg(dst.get_symbol(), p); +} - if( to_non_generic ) { - switch( space.get_type() ) { - case shared_space: to_addr_hw = generic_to_shared( smid, from_addr_hw ); break; - case local_space: to_addr_hw = generic_to_local( smid, hwtid, from_addr_hw ); break; - case global_space: to_addr_hw = generic_to_global(from_addr_hw ); break; - default: abort(); - } - } else { - switch( space.get_type() ) { - case shared_space: to_addr_hw = shared_to_generic( smid, from_addr_hw ); break; - case local_space: to_addr_hw = local_to_generic( smid, hwtid, from_addr_hw ) - + thread->get_local_mem_stack_pointer(); break; // add stack ptr here so that it can be passed as a pointer at function call - case global_space: to_addr_hw = global_to_generic( from_addr_hw ); break; - default: abort(); - } - } - - ptx_reg_t to_addr; - to_addr.u64 = to_addr_hw; - thread->set_reg(dst.get_symbol(),to_addr); -} - -void div_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - - - switch ( i_type ) { - case S8_TYPE: - data.s8 = src1_data.s8 / src2_data.s8 ; break; - case S16_TYPE: - data.s16 = src1_data.s16 / src2_data.s16; break; - case S32_TYPE: - data.s32 = src1_data.s32 / src2_data.s32; break; - case S64_TYPE: - data.s64 = src1_data.s64 / src2_data.s64; break; - case U8_TYPE: - data.u8 = src1_data.u8 / src2_data.u8 ; break; - case U16_TYPE: - data.u16 = src1_data.u16 / src2_data.u16; break; - case U32_TYPE: - data.u32 = src1_data.u32 / src2_data.u32; break; - case U64_TYPE: - data.u64 = src1_data.u64 / src2_data.u64; break; - case B8_TYPE: - data.u8 = src1_data.u8 / src2_data.u8 ; break; - case B16_TYPE: - data.u16 = src1_data.u16 / src2_data.u16; break; - case B32_TYPE: - data.u32 = src1_data.u32 / src2_data.u32; break; - case B64_TYPE: - data.u64 = src1_data.u64 / src2_data.u64; break; - case F16_TYPE: data.f16 = src1_data.f16 / src2_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = src1_data.f32 / src2_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 / src2_data.f64; break; - default: assert(0); break; - } - thread->set_operand_value(dst,data, i_type, thread,pI); -} - -void dp4a_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - printf("DP4A instruction not implemented yet"); - assert(0); - -} - -void ex2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case F32_TYPE: - data.f32 = cuda_math::__powf(2.0, src1_data.f32); +void decode_space(memory_space_t &space, ptx_thread_info *thread, + const operand_info &op, memory_space *&mem, addr_t &addr) { + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); + + if (space == param_space_unclassified) { + // need to op to determine whether it refers to a kernel param or local + // param + const symbol *s = op.get_symbol(); + const type_info *t = s->type(); + type_info_key ti = t->get_key(); + if (ti.is_param_kernel()) + space = param_space_kernel; + else if (ti.is_param_local()) { + space = param_space_local; + } + // mov r1, param-label + else if (ti.is_reg()) { + space = param_space_kernel; + } else { + printf("GPGPU-Sim PTX: ERROR ** cannot resolve .param space for '%s'\n", + s->name().c_str()); + abort(); + } + } + switch (space.get_type()) { + case global_space: + mem = thread->get_global_memory(); break; - default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); + case param_space_local: + case local_space: + mem = thread->m_local_mem; + addr += thread->get_local_mem_stack_pointer(); + break; + case tex_space: + mem = thread->get_tex_memory(); + break; + case surf_space: + mem = thread->get_surf_memory(); + break; + case param_space_kernel: + mem = thread->get_param_memory(); + break; + case shared_space: + mem = thread->m_shared_mem; break; - } - - thread->set_operand_value(dst,data, i_type, thread,pI); + case sstarr_space: + mem = thread->m_sstarr_mem; + break; + case const_space: + mem = thread->get_global_memory(); + break; + case generic_space: + if (thread->get_ptx_version().ver() >= 2.0) { + // convert generic address to memory space address + space = whichspace(addr); + switch (space.get_type()) { + case global_space: + mem = thread->get_global_memory(); + addr = generic_to_global(addr); + break; + case local_space: + mem = thread->m_local_mem; + addr = generic_to_local(smid, hwtid, addr); + break; + case shared_space: + mem = thread->m_shared_mem; + addr = generic_to_shared(smid, addr); + break; + default: + abort(); + } + } else { + abort(); + } + break; + case param_space_unclassified: + case undefined_space: + default: + abort(); + } } -void exit_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - thread->set_done(); - thread->exitCore(); - thread->registerExit(); +void ld_exec(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned type = pI->get_type(); + + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, type, thread, 1); + ptx_reg_t data; + memory_space_t space = pI->get_space(); + unsigned vector_spec = pI->get_vector(); + + memory_space *mem = NULL; + addr_t addr = src1_data.u32; + + decode_space(space, thread, src1, mem, addr); + + size_t size; + int t; + data.u64 = 0; + type_info_key::type_decode(type, size, t); + if (!vector_spec) { + mem->read(addr, size / 8, &data.s64); + if (type == S16_TYPE || type == S32_TYPE) sign_extend(data, size, dst); + thread->set_operand_value(dst, data, type, thread, pI); + } else { + ptx_reg_t data1, data2, data3, data4; + mem->read(addr, size / 8, &data1.s64); + mem->read(addr + size / 8, size / 8, &data2.s64); + if (vector_spec != V2_TYPE) { // either V3 or V4 + mem->read(addr + 2 * size / 8, size / 8, &data3.s64); + if (vector_spec != V3_TYPE) { // v4 + mem->read(addr + 3 * size / 8, size / 8, &data4.s64); + thread->set_vector_operand_values(dst, data1, data2, data3, data4); + } else // v3 + thread->set_vector_operand_values(dst, data1, data2, data3, data3); + } else // v2 + thread->set_vector_operand_values(dst, data1, data2, data2, data2); + } + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; } -void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry = false ); - -void fma_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - mad_def(pI,thread); +void ld_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ld_exec(pI, thread); } +void ldu_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ld_exec(pI, thread); +} + +void mma_st_impl(const ptx_instruction *pI, core_t *core, warp_inst_t &inst) { + size_t size; + unsigned smid; + int t; + int thrd, k; + ptx_thread_info *thread; + + const operand_info &src = pI->operand_lookup(1); + const operand_info &src1 = pI->operand_lookup(0); + const operand_info &src2 = pI->operand_lookup(2); + int tid; + unsigned type = pI->get_type(); + unsigned wmma_type = pI->get_wmma_type(); + unsigned wmma_layout = pI->get_wmma_layout(0); + int stride; + + if (core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); + + _memory_op_t insn_memory_op = + pI->has_memory_read() ? memory_load : memory_store; + for (thrd = 0; thrd < core->get_warp_size(); thrd++) { + thread = core->get_thread_info()[tid + thrd]; + ptx_reg_t addr_reg = thread->get_operand_value(src1, src, type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, src, type, thread, 1); + const operand_info &src_a = pI->operand_lookup(1); + unsigned nelem = src_a.get_vect_nelem(); + ptx_reg_t *v = new ptx_reg_t[8]; + thread->get_vector_operand_values(src_a, v, nelem); + stride = src2_data.u32; + + memory_space_t space = pI->get_space(); + + memory_space *mem = NULL; + addr_t addr = addr_reg.u32; + + new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; + int num_mem_txn = 0; -void isspacep_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a; - bool t=false; + smid = thread->get_hw_sid(); + if (whichspace(addr) == shared_space) { + addr = generic_to_shared(smid, addr); + space = shared_space; + } + decode_space(space, thread, src1, mem, addr); + + type_info_key::type_decode(type, size, t); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma_st: thrd=%d, addr=%x, fp(size=%zu), stride=%d\n", thrd, + addr_reg.u32, size, src2_data.u32); + addr_t new_addr = + addr + + thread_group_offset(thrd, wmma_type, wmma_layout, type, stride) * size / + 8; + addr_t push_addr; + + ptx_reg_t nw_v[8]; + for (k = 0; k < 8; k++) { + if (k % 2 == 0) + nw_v[k].s64 = (v[k / 2].s64 & 0xffff); + else + nw_v[k].s64 = ((v[k / 2].s64 & 0xffff0000) >> 16); + } - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - memory_space_t space = pI->get_space(); + for (k = 0; k < 8; k++) { + if (type == F32_TYPE) { + // mem->write(new_addr+4*acc_float_offset(k,wmma_layout,stride),size/8,&v[k].s64,thread,pI); + push_addr = new_addr + 4 * acc_float_offset(k, wmma_layout, stride); + mem->write(push_addr, size / 8, &v[k].s64, thread, pI); + mem_txn_addr[num_mem_txn++] = push_addr; + + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf( + "wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n", + thrd, v[0].s64, v[1].s64, v[2].s64, v[3].s64, v[4].s64, v[5].s64, + v[6].s64, v[7].s64); + float temp; + int l; + printf("thread=%d:", thrd); + for (l = 0; l < 8; l++) { + temp = v[l].f32; + printf("%.2f", temp); + } + printf("\n"); + } + } else if (type == F16_TYPE) { + if (wmma_layout == ROW) { + // mem->write(new_addr+k*2,size/8,&nw_v[k].s64,thread,pI); + push_addr = new_addr + k * 2; + mem->write(push_addr, size / 8, &nw_v[k].s64, thread, pI); + if (k % 2 == 0) mem_txn_addr[num_mem_txn++] = push_addr; + } else if (wmma_layout == COL) { + // mem->write(new_addr+k*2*stride,size/8,&nw_v[k].s64,thread,pI); + push_addr = new_addr + k * 2 * stride; + mem->write(push_addr, size / 8, &nw_v[k].s64, thread, pI); + mem_txn_addr[num_mem_txn++] = push_addr; + } - a = thread->get_reg(src1.get_symbol()); - addr_t addr = (addr_t)a.u64; - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf( + "wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n", + thrd, nw_v[0].s64, nw_v[1].s64, nw_v[2].s64, nw_v[3].s64, + nw_v[4].s64, nw_v[5].s64, nw_v[6].s64, nw_v[7].s64); + } + } - switch( space.get_type() ) { - case shared_space: t = isspace_shared( smid, addr ); - case local_space: t = isspace_local( smid, hwtid, addr ); - case global_space: t = isspace_global( addr ); - default: abort(); - } + delete[] v; + inst.space = space; + inst.set_addr(thrd, (new_addr_type *)mem_txn_addr, num_mem_txn); - ptx_reg_t p; - p.pred = t?1:0; + if ((type == F16_TYPE) && + (wmma_layout == COL)) // check the profiling xls for details + inst.data_size = 2; // 2 byte transaction + else + inst.data_size = 4; // 4 byte transaction - thread->set_reg(dst.get_symbol(),p); + assert(inst.memory_op == insn_memory_op); + // thread->m_last_effective_address = addr; + // thread->m_last_memory_space = space; + } } -void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand_info &op, memory_space *&mem, addr_t &addr) -{ - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); +void mma_ld_impl(const ptx_instruction *pI, core_t *core, warp_inst_t &inst) { + size_t size; + int t, i; + unsigned smid; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned type = pI->get_type(); + unsigned wmma_type = pI->get_wmma_type(); + unsigned wmma_layout = pI->get_wmma_layout(0); + int tid; + int thrd, stride; + ptx_thread_info *thread; + + if (core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); + + _memory_op_t insn_memory_op = + pI->has_memory_read() ? memory_load : memory_store; + + for (thrd = 0; thrd < core->get_warp_size(); thrd++) { + thread = core->get_thread_info()[tid + thrd]; + ptx_reg_t src1_data = + thread->get_operand_value(src1, dst, U32_TYPE, thread, 1); + ptx_reg_t src2_data = + thread->get_operand_value(src2, dst, U32_TYPE, thread, 1); + stride = src2_data.u32; + memory_space_t space = pI->get_space(); + + memory_space *mem = NULL; + addr_t addr = src1_data.u32; + smid = thread->get_hw_sid(); + if (whichspace(addr) == shared_space) { + addr = generic_to_shared(smid, addr); + space = shared_space; + } - if( space == param_space_unclassified ) { - // need to op to determine whether it refers to a kernel param or local param - const symbol *s = op.get_symbol(); - const type_info *t = s->type(); - type_info_key ti = t->get_key(); - if( ti.is_param_kernel() ) - space = param_space_kernel; - else if( ti.is_param_local() ) { - space = param_space_local; + decode_space(space, thread, src1, mem, addr); + type_info_key::type_decode(type, size, t); + + ptx_reg_t data[16]; + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma_ld: thrd=%d,addr=%x, fpsize=%zu, stride=%d\n", thrd, + src1_data.u32, size, src2_data.u32); + + addr_t new_addr = + addr + + thread_group_offset(thrd, wmma_type, wmma_layout, type, stride) * size / + 8; + addr_t fetch_addr; + new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; + int num_mem_txn = 0; + + if (wmma_type == LOAD_A) { + for (i = 0; i < 16; i++) { + if (wmma_layout == ROW) { + // mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr = new_addr + 2 * i; + mem->read(fetch_addr, size / 8, &data[i].s64); + } else if (wmma_layout == COL) { + // mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); + fetch_addr = new_addr + 2 * (i % 4) + 2 * stride * 4 * (i / 4); + mem->read(fetch_addr, size / 8, &data[i].s64); + } else { + printf("mma_ld:wrong_layout_type\n"); + abort(); + } + if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; } - //mov r1, param-label - else if (ti.is_reg() ){ - space = param_space_kernel; + } else if (wmma_type == LOAD_B) { + for (i = 0; i < 16; i++) { + if (wmma_layout == COL) { + // mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr = new_addr + 2 * i; + mem->read(fetch_addr, size / 8, &data[i].s64); + } else if (wmma_layout == ROW) { + // mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); + fetch_addr = new_addr + 2 * (i % 4) + 2 * stride * 4 * (i / 4); + mem->read(fetch_addr, size / 8, &data[i].s64); + } else { + printf("mma_ld:wrong_layout_type\n"); + abort(); + } + if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; } - else { - printf("GPGPU-Sim PTX: ERROR ** cannot resolve .param space for '%s'\n", s->name().c_str() ); - abort(); + } else if (wmma_type == LOAD_C) { + for (i = 0; i < 8; i++) { + if (type == F16_TYPE) { + if (wmma_layout == ROW) { + // mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr = new_addr + 2 * i; + mem->read(fetch_addr, size / 8, &data[i].s64); + if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; + } else if (wmma_layout == COL) { + // mem->read(new_addr+2*stride*i,size/8,&data[i].s64); + fetch_addr = new_addr + 2 * stride * i; + mem->read(fetch_addr, size / 8, &data[i].s64); + mem_txn_addr[num_mem_txn++] = fetch_addr; + } else { + printf("mma_ld:wrong_type\n"); + abort(); + } + } else if (type == F32_TYPE) { + // mem->read(new_addr+4*acc_float_offset(i,wmma_layout,stride),size/8,&data[i].s64); + fetch_addr = new_addr + 4 * acc_float_offset(i, wmma_layout, stride); + mem->read(fetch_addr, size / 8, &data[i].s64); + mem_txn_addr[num_mem_txn++] = fetch_addr; + } else { + printf("wrong type"); + abort(); + } } - } - switch ( space.get_type() ) { - case global_space: mem = thread->get_global_memory(); break; - case param_space_local: - case local_space: - mem = thread->m_local_mem; - addr += thread->get_local_mem_stack_pointer(); - break; - case tex_space: mem = thread->get_tex_memory(); break; - case surf_space: mem = thread->get_surf_memory(); break; - case param_space_kernel: mem = thread->get_param_memory(); break; - case shared_space: mem = thread->m_shared_mem; break; - case sstarr_space: mem = thread->m_sstarr_mem; break; - case const_space: mem = thread->get_global_memory(); break; - case generic_space: - if( thread->get_ptx_version().ver() >= 2.0 ) { - // convert generic address to memory space address - space = whichspace(addr); - switch ( space.get_type() ) { - case global_space: mem = thread->get_global_memory(); addr = generic_to_global(addr); break; - case local_space: mem = thread->m_local_mem; addr = generic_to_local(smid,hwtid,addr); break; - case shared_space: mem = thread->m_shared_mem; addr = generic_to_shared(smid,addr); break; - default: abort(); - } + } else { + printf("wrong wmma type\n"); + ; + abort(); + } + // generate timing memory request + inst.space = space; + inst.set_addr(thrd, (new_addr_type *)mem_txn_addr, num_mem_txn); + + if ((wmma_type == LOAD_C) && (type == F16_TYPE) && + (wmma_layout == COL)) // memory address is scattered, check the + // profiling xls for more detail. + inst.data_size = 2; // 2 byte transaction + else + inst.data_size = 4; // 4 byte transaction + assert(inst.memory_op == insn_memory_op); + + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + if (type == F16_TYPE) { + printf("\nmma_ld:thread%d= ", thrd); + for (i = 0; i < 16; i++) { + printf("%llx ", data[i].u64); + } + printf("\n"); + + printf("\nmma_ld:thread%d= ", thrd); + float temp; + for (i = 0; i < 16; i++) { + temp = data[i].f16; + printf("%.2f ", temp); + } + printf("\n"); } else { - abort(); + printf("\nmma_ld:thread%d= ", thrd); + for (i = 0; i < 8; i++) { + printf("%.2f ", data[i].f32); + } + printf("\n"); + printf("\nmma_ld:thread%d= ", thrd); + for (i = 0; i < 8; i++) { + printf("%llx ", data[i].u64); + } + printf("\n"); } + } + + if ((wmma_type == LOAD_C) && (type == F32_TYPE)) { + thread->set_wmma_vector_operand_values(dst, data[0], data[1], data[2], + data[3], data[4], data[5], data[6], + data[7]); + } else { + ptx_reg_t nw_data[8]; + int num_reg; + + if (wmma_type == LOAD_C) + num_reg = 4; + else + num_reg = 8; + + for (i = 0; i < num_reg; i++) { + nw_data[i].s64 = ((data[2 * i].s64 & 0xffff) << 16) | + ((data[2 * i + 1].s64 & 0xffff)); + } + + if (wmma_type == LOAD_C) + thread->set_vector_operand_values(dst, nw_data[0], nw_data[1], + nw_data[2], nw_data[3]); + else + thread->set_wmma_vector_operand_values( + dst, nw_data[0], nw_data[1], nw_data[2], nw_data[3], nw_data[4], + nw_data[5], nw_data[6], nw_data[7]); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf( + "mma_ld:data[0].s64=%llx,data[1].s64=%llx,new_data[0].s64=%llx\n", + data[0].u64, data[1].u64, nw_data[0].u64); + printf( + "mma_ld:data[2].s64=%llx,data[3].s64=%llx,new_data[1].s64=%llx\n", + data[2].u64, data[3].u64, nw_data[1].u64); + printf( + "mma_ld:data[4].s64=%llx,data[5].s64=%llx,new_data[2].s64=%llx\n", + data[4].u64, data[5].u64, nw_data[2].u64); + printf( + "mma_ld:data[6].s64=%llx,data[7].s64=%llx,new_data[3].s64=%llx\n", + data[6].u64, data[7].u64, nw_data[3].u64); + if (wmma_type != LOAD_C) { + printf( + "mma_ld:data[8].s64=%llx,data[9].s64=%llx,new_data[4].s64=%llx\n", + data[8].u64, data[9].u64, nw_data[4].s64); + printf( + "mma_ld:data[10].s64=%llx,data[11].s64=%llx,new_data[5].s64=%" + "llx\n", + data[10].u64, data[11].u64, nw_data[5].u64); + printf( + "mma_ld:data[12].s64=%llx,data[13].s64=%llx,new_data[6].s64=%" + "llx\n", + data[12].u64, data[13].u64, nw_data[6].u64); + printf( + "mma_ld:data[14].s64=%llx,data[15].s64=%llx,new_data[7].s64=%" + "llx\n", + data[14].u64, data[15].u64, nw_data[3].u64); + } + } + } + + // thread->m_last_effective_address = addr; + // thread->m_last_memory_space = space; + } +} + +void lg2_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch (i_type) { + case F32_TYPE: + d.f32 = log(a.f32) / log(2); break; - case param_space_unclassified: - case undefined_space: - default: - abort(); - } -} - -void ld_exec( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned type = pI->get_type(); - - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, type, thread, 1); - ptx_reg_t data; - memory_space_t space = pI->get_space(); - unsigned vector_spec = pI->get_vector(); - - memory_space *mem = NULL; - addr_t addr = src1_data.u32; - - decode_space(space,thread,src1,mem,addr); - - size_t size; - int t; - data.u64=0; - type_info_key::type_decode(type,size,t); - if (!vector_spec) { - mem->read(addr,size/8,&data.s64); - if( type == S16_TYPE || type == S32_TYPE ) - sign_extend(data,size,dst); - thread->set_operand_value(dst,data, type, thread, pI); - } else { - ptx_reg_t data1, data2, data3, data4; - mem->read(addr,size/8,&data1.s64); - mem->read(addr+size/8,size/8,&data2.s64); - if (vector_spec != V2_TYPE) { //either V3 or V4 - mem->read(addr+2*size/8,size/8,&data3.s64); - if (vector_spec != V3_TYPE) { //v4 - mem->read(addr+3*size/8,size/8,&data4.s64); - thread->set_vector_operand_values(dst,data1,data2,data3,data4); - } else //v3 - thread->set_vector_operand_values(dst,data1,data2,data3,data3); - } else //v2 - thread->set_vector_operand_values(dst,data1,data2,data2,data2); - } - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; -} - -void ld_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ld_exec(pI,thread); -} -void ldu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ld_exec(pI,thread); -} - -void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t &inst ) -{ - size_t size; - unsigned smid; - int t; - int thrd, k; - ptx_thread_info *thread; - - const operand_info &src = pI->operand_lookup(1); - const operand_info &src1 = pI->operand_lookup(0); - const operand_info &src2 = pI->operand_lookup(2); - int tid ; - unsigned type = pI->get_type(); - unsigned wmma_type = pI->get_wmma_type(); - unsigned wmma_layout = pI->get_wmma_layout(0); - int stride; - - if(core->get_gpu()->is_functional_sim()) - tid= inst.warp_id_func()*core->get_warp_size(); - else - tid= inst.warp_id()*core->get_warp_size(); - - _memory_op_t insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; - for (thrd=0; thrd < core->get_warp_size(); thrd++) { - thread = core->get_thread_info()[tid+thrd]; - ptx_reg_t addr_reg = thread->get_operand_value(src1, src, type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, src, type, thread, 1); - const operand_info &src_a= pI->operand_lookup(1); - unsigned nelem = src_a.get_vect_nelem(); - ptx_reg_t* v= new ptx_reg_t[8]; - thread->get_vector_operand_values( src_a, v, nelem ); - stride = src2_data.u32; - - memory_space_t space = pI->get_space(); - - memory_space *mem = NULL; - addr_t addr = addr_reg.u32; - - new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; - int num_mem_txn=0; - - smid = thread->get_hw_sid(); - if( whichspace(addr) == shared_space ) { - addr= generic_to_shared(smid,addr); - space = shared_space; - } - decode_space(space,thread,src1,mem,addr); - - type_info_key::type_decode(type, size, t); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma_st: thrd=%d, addr=%x, fp(size=%zu), stride=%d\n", thrd, addr_reg.u32, size, src2_data.u32); - addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8; - addr_t push_addr; - - ptx_reg_t nw_v[8]; - for(k=0;k<8;k++){ - if(k%2==0) - nw_v[k].s64=(v[k/2].s64&0xffff); - else - nw_v[k].s64=((v[k/2].s64&0xffff0000)>>16); - } - - for(k=0;k<8;k++){ - if(type==F32_TYPE){ - //mem->write(new_addr+4*acc_float_offset(k,wmma_layout,stride),size/8,&v[k].s64,thread,pI); - push_addr=new_addr+4*acc_float_offset(k,wmma_layout,stride); - mem->write(push_addr,size/8,&v[k].s64,thread,pI); - mem_txn_addr[num_mem_txn++]=push_addr; - - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n",thrd,v[0].s64,v[1].s64,v[2].s64,v[3].s64,v[4].s64,v[5].s64,v[6].s64,v[7].s64); - float temp; - int l; - printf("thread=%d:",thrd); - for(l=0;l<8;l++){ - temp=v[l].f32; - printf("%.2f",temp); - } - printf("\n"); - } - } - else if(type==F16_TYPE){ - if(wmma_layout==ROW){ - //mem->write(new_addr+k*2,size/8,&nw_v[k].s64,thread,pI); - push_addr=new_addr+k*2; - mem->write(push_addr,size/8,&nw_v[k].s64,thread,pI); - if(k%2==0) - mem_txn_addr[num_mem_txn++]=push_addr; - } - else if(wmma_layout==COL){ - //mem->write(new_addr+k*2*stride,size/8,&nw_v[k].s64,thread,pI); - push_addr=new_addr+k*2*stride; - mem->write(push_addr,size/8,&nw_v[k].s64,thread,pI); - mem_txn_addr[num_mem_txn++]=push_addr; - } - - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n",thrd,nw_v[0].s64,nw_v[1].s64,nw_v[2].s64,nw_v[3].s64,nw_v[4].s64,nw_v[5].s64,nw_v[6].s64,nw_v[7].s64); - } - } - - delete [] v; - inst.space = space; - inst.set_addr(thrd, (new_addr_type *)mem_txn_addr , num_mem_txn); - - if((type==F16_TYPE)&&(wmma_layout==COL))//check the profiling xls for details - inst.data_size = 2; // 2 byte transaction - else - inst.data_size = 4; // 4 byte transaction - - assert( inst.memory_op == insn_memory_op ); - //thread->m_last_effective_address = addr; - //thread->m_last_memory_space = space; - } -} - -void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t &inst ) -{ - size_t size; - int t,i; - unsigned smid; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned type = pI->get_type(); - unsigned wmma_type = pI->get_wmma_type(); - unsigned wmma_layout = pI->get_wmma_layout(0); - int tid; - int thrd,stride; - ptx_thread_info *thread; - - - if(core->get_gpu()->is_functional_sim()) - tid= inst.warp_id_func()*core->get_warp_size(); - else - tid= inst.warp_id()*core->get_warp_size(); - - _memory_op_t insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; - - for (thrd=0; thrd < core->get_warp_size(); thrd++){ - thread = core->get_thread_info()[tid+thrd]; - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, U32_TYPE, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, dst, U32_TYPE, thread, 1); - stride=src2_data.u32; - memory_space_t space = pI->get_space(); - - memory_space *mem = NULL; - addr_t addr = src1_data.u32; - smid = thread->get_hw_sid(); - if( whichspace(addr) == shared_space ) { - addr= generic_to_shared(smid,addr); - space = shared_space; - } - - decode_space(space,thread,src1,mem,addr); - type_info_key::type_decode(type, size, t); - - ptx_reg_t data[16]; - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma_ld: thrd=%d,addr=%x, fpsize=%zu, stride=%d\n", thrd, src1_data.u32, size, src2_data.u32); - - addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8; - addr_t fetch_addr; - new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; - int num_mem_txn=0; - - if(wmma_type==LOAD_A){ - for(i=0;i<16;i++){ - if(wmma_layout==ROW){ - //mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr=new_addr+2*i; - mem->read(fetch_addr,size/8,&data[i].s64); - } - else if(wmma_layout==COL){ - //mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); - fetch_addr=new_addr+2*(i%4)+2*stride*4*(i/4); - mem->read(fetch_addr,size/8,&data[i].s64); - } - else{ - printf("mma_ld:wrong_layout_type\n"); - abort(); - - } - if(i%2==0) - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - } - else if(wmma_type==LOAD_B){ - for(i=0;i<16;i++){ - if(wmma_layout==COL){ - //mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr=new_addr+2*i; - mem->read(fetch_addr,size/8,&data[i].s64); - } - else if(wmma_layout==ROW){ - //mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); - fetch_addr=new_addr+2*(i%4)+2*stride*4*(i/4); - mem->read(fetch_addr,size/8,&data[i].s64); - } - else{ - printf("mma_ld:wrong_layout_type\n"); - abort(); - } - if(i%2==0) - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - } - else if(wmma_type==LOAD_C){ - for(i=0;i<8;i++){ - if(type==F16_TYPE){ - if(wmma_layout==ROW){ - //mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr=new_addr+2*i; - mem->read(fetch_addr,size/8,&data[i].s64); - if(i%2==0) - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - else if(wmma_layout==COL){ - //mem->read(new_addr+2*stride*i,size/8,&data[i].s64); - fetch_addr=new_addr+2*stride*i; - mem->read(fetch_addr,size/8,&data[i].s64); - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - else{ - printf("mma_ld:wrong_type\n"); - abort(); - } - } - else if(type==F32_TYPE){ - //mem->read(new_addr+4*acc_float_offset(i,wmma_layout,stride),size/8,&data[i].s64); - fetch_addr=new_addr+4*acc_float_offset(i,wmma_layout,stride); - mem->read(fetch_addr,size/8,&data[i].s64); - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - else{ - printf("wrong type"); - abort(); - } - } - } - else{ - printf("wrong wmma type\n");; - abort(); - } - //generate timing memory request - inst.space = space; - inst.set_addr(thrd, (new_addr_type *)mem_txn_addr , num_mem_txn); - - if((wmma_type==LOAD_C)&&(type==F16_TYPE)&&(wmma_layout==COL))//memory address is scattered, check the profiling xls for more detail. - inst.data_size = 2; // 2 byte transaction - else - inst.data_size = 4; // 4 byte transaction - assert( inst.memory_op == insn_memory_op ); - - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - if(type==F16_TYPE){ - printf("\nmma_ld:thread%d= ",thrd); - for(i=0;i<16;i++){ - printf("%llx ",data[i].u64); - } - printf("\n"); - - printf("\nmma_ld:thread%d= ",thrd); - float temp; - for(i=0;i<16;i++){ - temp=data[i].f16; - printf("%.2f ",temp); - } - printf("\n"); - } - else{ - printf("\nmma_ld:thread%d= ",thrd); - for(i=0;i<8;i++){ - printf("%.2f ",data[i].f32); - } - printf("\n"); - printf("\nmma_ld:thread%d= ",thrd); - for(i=0;i<8;i++){ - printf("%llx ",data[i].u64); - } - printf("\n"); - } - } - - if((wmma_type==LOAD_C)&&(type==F32_TYPE)){ - thread->set_wmma_vector_operand_values(dst,data[0],data[1],data[2],data[3],data[4],data[5],data[6],data[7]); - } - else{ - ptx_reg_t nw_data[8]; - int num_reg; - - if(wmma_type==LOAD_C) - num_reg=4; - else - num_reg=8; - - for(i=0;iset_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3]); - else - thread->set_wmma_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3],nw_data[4],nw_data[5],nw_data[6],nw_data[7]); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("mma_ld:data[0].s64=%llx,data[1].s64=%llx,new_data[0].s64=%llx\n",data[0].u64,data[1].u64,nw_data[0].u64); - printf("mma_ld:data[2].s64=%llx,data[3].s64=%llx,new_data[1].s64=%llx\n",data[2].u64,data[3].u64,nw_data[1].u64); - printf("mma_ld:data[4].s64=%llx,data[5].s64=%llx,new_data[2].s64=%llx\n",data[4].u64,data[5].u64,nw_data[2].u64); - printf("mma_ld:data[6].s64=%llx,data[7].s64=%llx,new_data[3].s64=%llx\n",data[6].u64,data[7].u64,nw_data[3].u64); - if(wmma_type!=LOAD_C){ - printf("mma_ld:data[8].s64=%llx,data[9].s64=%llx,new_data[4].s64=%llx\n",data[8].u64,data[9].u64,nw_data[4].s64); - printf("mma_ld:data[10].s64=%llx,data[11].s64=%llx,new_data[5].s64=%llx\n",data[10].u64,data[11].u64,nw_data[5].u64); - printf("mma_ld:data[12].s64=%llx,data[13].s64=%llx,new_data[6].s64=%llx\n",data[12].u64,data[13].u64,nw_data[6].u64); - printf("mma_ld:data[14].s64=%llx,data[15].s64=%llx,new_data[7].s64=%llx\n",data[14].u64,data[15].u64,nw_data[3].u64); - } - } - } - - //thread->m_last_effective_address = addr; - //thread->m_last_memory_space = space; - } -} - -void lg2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case F32_TYPE: - d.f32 = log(a.f32)/log(2); - break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void mad24_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t d, t; +void mad24_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t d, t; - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - unsigned sat_mode = pI->saturation_mode(); + unsigned sat_mode = pI->saturation_mode(); - assert( !pI->is_wide() ); + assert(!pI->is_wide()); - switch ( i_type ) { - case S32_TYPE: + switch (i_type) { + case S32_TYPE: t.s64 = a.s32 * b.s32; - if ( pI->is_hi() ) { - d.s64 = (t.s64>>16) + c.s32; - if ( sat_mode ) { - if ( d.s64 > (int)0x7FFFFFFF ) - d.s64 = (int)0x7FFFFFFF; - else if ( d.s64 < (int)0x80000000 ) - d.s64 = (int)0x80000000; - } - } else if ( pI->is_lo() ) d.s64 = t.s32 + c.s32; - else assert(0); - break; - case U32_TYPE: + if (pI->is_hi()) { + d.s64 = (t.s64 >> 16) + c.s32; + if (sat_mode) { + if (d.s64 > (int)0x7FFFFFFF) + d.s64 = (int)0x7FFFFFFF; + else if (d.s64 < (int)0x80000000) + d.s64 = (int)0x80000000; + } + } else if (pI->is_lo()) + d.s64 = t.s32 + c.s32; + else + assert(0); + break; + case U32_TYPE: t.u64 = a.u32 * b.u32; - if ( pI->is_hi() ) d.u64 = (t.u64>>16) + c.u32; - else if ( pI->is_lo() ) d.u64 = t.u32 + c.u32; - else assert(0); + if (pI->is_hi()) + d.u64 = (t.u64 >> 16) + c.u32; + else if (pI->is_lo()) + d.u64 = t.u32 + c.u32; + else + assert(0); break; - default: + default: assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void mad_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - mad_def(pI, thread, false); +void mad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + mad_def(pI, thread, false); } -void madp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - mad_def(pI, thread, true); +void madp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + mad_def(pI, thread, true); } -void madc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - mad_def(pI, thread, true); +void madc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + mad_def(pI, thread, true); } -void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t d, t; - - int carry=0; - int overflow=0; - - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - - // take the carry bit, it should be the 4th operand - ptx_reg_t carry_bit; - carry_bit.u64 = 0; - if (use_carry) { - const operand_info &carry = pI->operand_lookup(4); - carry_bit = thread->get_operand_value(carry, dst, PRED_TYPE, thread, 0); - carry_bit.pred &= 0x4; - carry_bit.pred >>=2; - } +void mad_def(const ptx_instruction *pI, ptx_thread_info *thread, + bool use_carry) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t d, t; + + int carry = 0; + int overflow = 0; + + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + + // take the carry bit, it should be the 4th operand + ptx_reg_t carry_bit; + carry_bit.u64 = 0; + if (use_carry) { + const operand_info &carry = pI->operand_lookup(4); + carry_bit = thread->get_operand_value(carry, dst, PRED_TYPE, thread, 0); + carry_bit.pred &= 0x4; + carry_bit.pred >>= 2; + } - unsigned rounding_mode = pI->rounding_mode(); + unsigned rounding_mode = pI->rounding_mode(); - switch ( i_type ) { - case S16_TYPE: + switch (i_type) { + case S16_TYPE: t.s32 = a.s16 * b.s16; - if ( pI->is_wide() ) d.s32 = t.s32 + c.s32 + carry_bit.pred; - else if ( pI->is_hi() ) d.s16 = (t.s32>>16) + c.s16 + carry_bit.pred; - else if ( pI->is_lo() ) d.s16 = t.s16 + c.s16 + carry_bit.pred; - else assert(0); - carry = ((long long int)(t.s32 + c.s32 + carry_bit.pred)&0x100000000)>>32; + if (pI->is_wide()) + d.s32 = t.s32 + c.s32 + carry_bit.pred; + else if (pI->is_hi()) + d.s16 = (t.s32 >> 16) + c.s16 + carry_bit.pred; + else if (pI->is_lo()) + d.s16 = t.s16 + c.s16 + carry_bit.pred; + else + assert(0); + carry = + ((long long int)(t.s32 + c.s32 + carry_bit.pred) & 0x100000000) >> 32; break; - case S32_TYPE: + case S32_TYPE: t.s64 = a.s32 * b.s32; - if ( pI->is_wide() ) d.s64 = t.s64 + c.s64 + carry_bit.pred; - else if ( pI->is_hi() ) d.s32 = (t.s64>>32) + c.s32 + carry_bit.pred; - else if ( pI->is_lo() ) d.s32 = t.s32 + c.s32 + carry_bit.pred; - else assert(0); + if (pI->is_wide()) + d.s64 = t.s64 + c.s64 + carry_bit.pred; + else if (pI->is_hi()) + d.s32 = (t.s64 >> 32) + c.s32 + carry_bit.pred; + else if (pI->is_lo()) + d.s32 = t.s32 + c.s32 + carry_bit.pred; + else + assert(0); break; - case S64_TYPE: + case S64_TYPE: t.s64 = a.s64 * b.s64; - assert( !pI->is_wide() ); - assert( !pI->is_hi() ); - assert( use_carry == false); - if ( pI->is_lo() ) d.s64 = t.s64 + c.s64 + carry_bit.pred; - else assert(0); + assert(!pI->is_wide()); + assert(!pI->is_hi()); + assert(use_carry == false); + if (pI->is_lo()) + d.s64 = t.s64 + c.s64 + carry_bit.pred; + else + assert(0); break; - case U16_TYPE: + case U16_TYPE: t.u32 = a.u16 * b.u16; - if ( pI->is_wide() ) d.u32 = t.u32 + c.u32 + carry_bit.pred; - else if ( pI->is_hi() ) d.u16 = (t.u32 + c.u16 + carry_bit.pred)>>16; - else if ( pI->is_lo() ) d.u16 = t.u16 + c.u16 + carry_bit.pred; - else assert(0); - carry = ((long long int)((long long int)t.u32 + c.u32 + carry_bit.pred)&0x100000000)>>32; + if (pI->is_wide()) + d.u32 = t.u32 + c.u32 + carry_bit.pred; + else if (pI->is_hi()) + d.u16 = (t.u32 + c.u16 + carry_bit.pred) >> 16; + else if (pI->is_lo()) + d.u16 = t.u16 + c.u16 + carry_bit.pred; + else + assert(0); + carry = ((long long int)((long long int)t.u32 + c.u32 + carry_bit.pred) & + 0x100000000) >> + 32; break; - case U32_TYPE: + case U32_TYPE: t.u64 = a.u32 * b.u32; - if ( pI->is_wide() ) d.u64 = t.u64 + c.u64 + carry_bit.pred; - else if ( pI->is_hi() ) d.u32 = (t.u64 + c.u32 + carry_bit.pred)>>32; - else if ( pI->is_lo() ) d.u32 = t.u32 + c.u32 + carry_bit.pred; - else assert(0); + if (pI->is_wide()) + d.u64 = t.u64 + c.u64 + carry_bit.pred; + else if (pI->is_hi()) + d.u32 = (t.u64 + c.u32 + carry_bit.pred) >> 32; + else if (pI->is_lo()) + d.u32 = t.u32 + c.u32 + carry_bit.pred; + else + assert(0); break; - case U64_TYPE: + case U64_TYPE: t.u64 = a.u64 * b.u64; - assert( !pI->is_wide() ); - assert( !pI->is_hi() ); - assert( use_carry == false); - if ( pI->is_lo() ) d.u64 = t.u64 + c.u64 + carry_bit.pred; - else assert(0); - break; - case F16_TYPE:{ - // assert(0); - // break; - assert( use_carry == false); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - d.f16 = a.f16 * b.f16 + c.f16; - if ( pI->saturation_mode() ) { - if ( d.f16 < 0 ) d.f16 = 0; - else if ( d.f16 > 1.0f ) d.f16 = 1.0f; - } - fesetround( orig_rm ); - break; - } - case F32_TYPE: { - assert( use_carry == false); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - d.f32 = a.f32 * b.f32 + c.f32; - if ( pI->saturation_mode() ) { - if ( d.f32 < 0 ) d.f32 = 0; - else if ( d.f32 > 1.0f ) d.f32 = 1.0f; - } - fesetround( orig_rm ); - break; - } - case F64_TYPE: case FF64_TYPE: { - assert( use_carry == false); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - d.f64 = a.f64 * b.f64 + c.f64; - if ( pI->saturation_mode() ) { - if ( d.f64 < 0 ) d.f64 = 0; - else if ( d.f64 > 1.0f ) d.f64 = 1.0; - } - fesetround( orig_rm ); - break; + assert(!pI->is_wide()); + assert(!pI->is_hi()); + assert(use_carry == false); + if (pI->is_lo()) + d.u64 = t.u64 + c.u64 + carry_bit.pred; + else + assert(0); + break; + case F16_TYPE: { + // assert(0); + // break; + assert(use_carry == false); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + d.f16 = a.f16 * b.f16 + c.f16; + if (pI->saturation_mode()) { + if (d.f16 < 0) + d.f16 = 0; + else if (d.f16 > 1.0f) + d.f16 = 1.0f; + } + fesetround(orig_rm); + break; + } + case F32_TYPE: { + assert(use_carry == false); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + d.f32 = a.f32 * b.f32 + c.f32; + if (pI->saturation_mode()) { + if (d.f32 < 0) + d.f32 = 0; + else if (d.f32 > 1.0f) + d.f32 = 1.0f; + } + fesetround(orig_rm); + break; + } + case F64_TYPE: + case FF64_TYPE: { + assert(use_carry == false); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; } - default: + d.f64 = a.f64 * b.f64 + c.f64; + if (pI->saturation_mode()) { + if (d.f64 < 0) + d.f64 = 0; + else if (d.f64 > 1.0f) + d.f64 = 1.0; + } + fesetround(orig_rm); + break; + } + default: assert(0); break; - } - thread->set_operand_value(dst, d, i_type, thread, pI, overflow, carry); -} - -bool isNaN(float x) -{ - return std::isnan(x); + } + thread->set_operand_value(dst, d, i_type, thread, pI, overflow, carry); } -bool isNaN(double x) -{ - return std::isnan(x); -} +bool isNaN(float x) { return std::isnan(x); } -void max_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +bool isNaN(double x) { return std::isnan(x); } - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); +void max_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case U16_TYPE: d.u16 = MY_MAX_I(a.u16,b.u16); break; - case U32_TYPE: d.u32 = MY_MAX_I(a.u32,b.u32); break; - case U64_TYPE: d.u64 = MY_MAX_I(a.u64,b.u64); break; - case S16_TYPE: d.s16 = MY_MAX_I(a.s16,b.s16); break; - case S32_TYPE: d.s32 = MY_MAX_I(a.s32,b.s32); break; - case S64_TYPE: d.s64 = MY_MAX_I(a.s64,b.s64); break; - case F32_TYPE: d.f32 = MY_MAX_F(a.f32,b.f32); break; - case F64_TYPE: case FF64_TYPE: d.f64 = MY_MAX_F(a.f64,b.f64); break; - default: + switch (i_type) { + case U16_TYPE: + d.u16 = MY_MAX_I(a.u16, b.u16); + break; + case U32_TYPE: + d.u32 = MY_MAX_I(a.u32, b.u32); + break; + case U64_TYPE: + d.u64 = MY_MAX_I(a.u64, b.u64); + break; + case S16_TYPE: + d.s16 = MY_MAX_I(a.s16, b.s16); + break; + case S32_TYPE: + d.s32 = MY_MAX_I(a.s32, b.s32); + break; + case S64_TYPE: + d.s64 = MY_MAX_I(a.s64, b.s64); + break; + case F32_TYPE: + d.f32 = MY_MAX_F(a.f32, b.f32); + break; + case F64_TYPE: + case FF64_TYPE: + d.f64 = MY_MAX_F(a.f64, b.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void membar_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - // handled by timing simulator +void membar_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // handled by timing simulator } -void min_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); +void min_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case U16_TYPE: d.u16 = MY_MIN_I(a.u16,b.u16); break; - case U32_TYPE: d.u32 = MY_MIN_I(a.u32,b.u32); break; - case U64_TYPE: d.u64 = MY_MIN_I(a.u64,b.u64); break; - case S16_TYPE: d.s16 = MY_MIN_I(a.s16,b.s16); break; - case S32_TYPE: d.s32 = MY_MIN_I(a.s32,b.s32); break; - case S64_TYPE: d.s64 = MY_MIN_I(a.s64,b.s64); break; - case F32_TYPE: d.f32 = MY_MIN_F(a.f32,b.f32); break; - case F64_TYPE: case FF64_TYPE: d.f64 = MY_MIN_F(a.f64,b.f64); break; - default: + switch (i_type) { + case U16_TYPE: + d.u16 = MY_MIN_I(a.u16, b.u16); + break; + case U32_TYPE: + d.u32 = MY_MIN_I(a.u32, b.u32); + break; + case U64_TYPE: + d.u64 = MY_MIN_I(a.u64, b.u64); + break; + case S16_TYPE: + d.s16 = MY_MIN_I(a.s16, b.s16); + break; + case S32_TYPE: + d.s32 = MY_MIN_I(a.s32, b.s32); + break; + case S64_TYPE: + d.s64 = MY_MIN_I(a.s64, b.s64); + break; + case F32_TYPE: + d.f32 = MY_MIN_F(a.f32, b.f32); + break; + case F64_TYPE: + case FF64_TYPE: + d.f64 = MY_MIN_F(a.f64, b.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void mov_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; +void mov_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - assert( src1.is_param_local() == 0 ); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + assert(src1.is_param_local() == 0); - if( (src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && (i_type != BB128_TYPE) && (i_type != FF64_TYPE) ) { - // pack or unpack operation - unsigned nbits_to_move; - ptx_reg_t tmp_bits; + if ((src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && + (i_type != BB128_TYPE) && (i_type != FF64_TYPE)) { + // pack or unpack operation + unsigned nbits_to_move; + ptx_reg_t tmp_bits; - switch( pI->get_type() ) { - case B16_TYPE: nbits_to_move = 16; break; - case B32_TYPE: nbits_to_move = 32; break; - case B64_TYPE: nbits_to_move = 64; break; - default: printf("Execution error: mov pack/unpack with unsupported type qualifier\n"); assert(0); break; - } + switch (pI->get_type()) { + case B16_TYPE: + nbits_to_move = 16; + break; + case B32_TYPE: + nbits_to_move = 32; + break; + case B64_TYPE: + nbits_to_move = 64; + break; + default: + printf( + "Execution error: mov pack/unpack with unsupported type " + "qualifier\n"); + assert(0); + break; + } - if( src1.is_vector() ) { - unsigned nelem = src1.get_vect_nelem(); - ptx_reg_t v[4]; - thread->get_vector_operand_values(src1, v, nelem ); - - unsigned bits_per_src_elem = nbits_to_move / nelem; - for( unsigned i=0; i < nelem; i++ ) { - switch(bits_per_src_elem) { - case 8: tmp_bits.u64 |= ((unsigned long long)(v[i].u8) << (8*i)); break; - case 16: tmp_bits.u64 |= ((unsigned long long)(v[i].u16) << (16*i)); break; - case 32: tmp_bits.u64 |= ((unsigned long long)(v[i].u32) << (32*i)); break; - default: printf("Execution error: mov pack/unpack with unsupported source/dst size ratio (src)\n"); assert(0); break; - } - } - } else { - data = thread->get_operand_value(src1, dst, i_type, thread, 1); - - switch( pI->get_type() ) { - case B16_TYPE: tmp_bits.u16 = data.u16; break; - case B32_TYPE: tmp_bits.u32 = data.u32; break; - case B64_TYPE: tmp_bits.u64 = data.u64; break; - default: assert(0); break; - } - } + if (src1.is_vector()) { + unsigned nelem = src1.get_vect_nelem(); + ptx_reg_t v[4]; + thread->get_vector_operand_values(src1, v, nelem); - if( dst.is_vector() ) { - unsigned nelem = dst.get_vect_nelem(); - ptx_reg_t v[4]; - unsigned bits_per_dst_elem = nbits_to_move / nelem; - for( unsigned i=0; i < nelem; i++ ) { - switch(bits_per_dst_elem) { - case 8: v[i].u8 = (tmp_bits.u64 >> (8*i)) & ((unsigned long long) 0xFF); break; - case 16: v[i].u16 = (tmp_bits.u64 >> (16*i)) & ((unsigned long long) 0xFFFF); break; - case 32: v[i].u32 = (tmp_bits.u64 >> (32*i)) & ((unsigned long long) 0xFFFFFFFF); break; - default: - printf("Execution error: mov pack/unpack with unsupported source/dst size ratio (dst)\n"); - assert(0); - break; - } - } - thread->set_vector_operand_values(dst,v[0],v[1],v[2],v[3]); - } else { - thread->set_operand_value(dst,tmp_bits, i_type, thread, pI); + unsigned bits_per_src_elem = nbits_to_move / nelem; + for (unsigned i = 0; i < nelem; i++) { + switch (bits_per_src_elem) { + case 8: + tmp_bits.u64 |= ((unsigned long long)(v[i].u8) << (8 * i)); + break; + case 16: + tmp_bits.u64 |= ((unsigned long long)(v[i].u16) << (16 * i)); + break; + case 32: + tmp_bits.u64 |= ((unsigned long long)(v[i].u32) << (32 * i)); + break; + default: + printf( + "Execution error: mov pack/unpack with unsupported source/dst " + "size ratio (src)\n"); + assert(0); + break; + } } - } else if (i_type == PRED_TYPE and src1.is_literal() == true) { - // in ptx, literal input translate to predicate as 0 = false and 1 = true - // we have adopted the opposite to simplify implementation of zero flags in ptxplus + } else { data = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t finaldata; - finaldata.pred = (data.u32 == 0)? 1 : 0; // setting zero-flag in predicate - thread->set_operand_value(dst, finaldata, i_type, thread, pI); - } else { - - data = thread->get_operand_value(src1, dst, i_type, thread, 1); + switch (pI->get_type()) { + case B16_TYPE: + tmp_bits.u16 = data.u16; + break; + case B32_TYPE: + tmp_bits.u32 = data.u32; + break; + case B64_TYPE: + tmp_bits.u64 = data.u64; + break; + default: + assert(0); + break; + } + } - thread->set_operand_value(dst, data, i_type, thread, pI); + if (dst.is_vector()) { + unsigned nelem = dst.get_vect_nelem(); + ptx_reg_t v[4]; + unsigned bits_per_dst_elem = nbits_to_move / nelem; + for (unsigned i = 0; i < nelem; i++) { + switch (bits_per_dst_elem) { + case 8: + v[i].u8 = (tmp_bits.u64 >> (8 * i)) & ((unsigned long long)0xFF); + break; + case 16: + v[i].u16 = + (tmp_bits.u64 >> (16 * i)) & ((unsigned long long)0xFFFF); + break; + case 32: + v[i].u32 = + (tmp_bits.u64 >> (32 * i)) & ((unsigned long long)0xFFFFFFFF); + break; + default: + printf( + "Execution error: mov pack/unpack with unsupported source/dst " + "size ratio (dst)\n"); + assert(0); + break; + } + } + thread->set_vector_operand_values(dst, v[0], v[1], v[2], v[3]); + } else { + thread->set_operand_value(dst, tmp_bits, i_type, thread, pI); + } + } else if (i_type == PRED_TYPE and src1.is_literal() == true) { + // in ptx, literal input translate to predicate as 0 = false and 1 = true + // we have adopted the opposite to simplify implementation of zero flags in + // ptxplus + data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + ptx_reg_t finaldata; + finaldata.pred = (data.u32 == 0) ? 1 : 0; // setting zero-flag in predicate + thread->set_operand_value(dst, finaldata, i_type, thread, pI); + } else { + data = thread->get_operand_value(src1, dst, i_type, thread, 1); - } + thread->set_operand_value(dst, data, i_type, thread, pI); + } } -void mul24_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void mul24_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - //src1_data = srcOperandModifiers(src1_data, src1, dst, i_type, thread); - //src2_data = srcOperandModifiers(src2_data, src2, dst, i_type, thread); + // src1_data = srcOperandModifiers(src1_data, src1, dst, i_type, thread); + // src2_data = srcOperandModifiers(src2_data, src2, dst, i_type, thread); - src1_data.mask_and(0,0x00FFFFFF); - src2_data.mask_and(0,0x00FFFFFF); + src1_data.mask_and(0, 0x00FFFFFF); + src2_data.mask_and(0, 0x00FFFFFF); - switch ( i_type ) { - case S32_TYPE: - if( src1_data.get_bit(23) ) - src1_data.mask_or(0xFFFFFFFF,0xFF000000); - if( src2_data.get_bit(23) ) - src2_data.mask_or(0xFFFFFFFF,0xFF000000); + switch (i_type) { + case S32_TYPE: + if (src1_data.get_bit(23)) src1_data.mask_or(0xFFFFFFFF, 0xFF000000); + if (src2_data.get_bit(23)) src2_data.mask_or(0xFFFFFFFF, 0xFF000000); data.s64 = src1_data.s64 * src2_data.s64; break; - case U32_TYPE: + case U32_TYPE: data.u64 = src1_data.u64 * src2_data.u64; break; - default: - printf("GPGPU-Sim PTX: Execution error - type mismatch with instruction\n"); + default: + printf( + "GPGPU-Sim PTX: Execution error - type mismatch with instruction\n"); assert(0); break; - } + } - if ( pI->is_hi() ) { - data.u64 = data.u64 >> 16; - data.mask_and(0,0xFFFFFFFF); - } else if (pI->is_lo()) { - data.mask_and(0,0xFFFFFFFF); - } + if (pI->is_hi()) { + data.u64 = data.u64 >> 16; + data.mask_and(0, 0xFFFFFFFF); + } else if (pI->is_lo()) { + data.mask_and(0, 0xFFFFFFFF); + } - thread->set_operand_value(dst, data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void mul_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; +void mul_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - ptx_reg_t d, t; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + ptx_reg_t d, t; - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - unsigned rounding_mode = pI->rounding_mode(); + unsigned rounding_mode = pI->rounding_mode(); - switch ( i_type ) { - case S16_TYPE: + switch (i_type) { + case S16_TYPE: t.s32 = ((int)a.s16) * ((int)b.s16); - if ( pI->is_wide() ) d.s32 = t.s32; - else if ( pI->is_hi() ) d.s16 = (t.s32>>16); - else if ( pI->is_lo() ) d.s16 = t.s16; - else assert(0); + if (pI->is_wide()) + d.s32 = t.s32; + else if (pI->is_hi()) + d.s16 = (t.s32 >> 16); + else if (pI->is_lo()) + d.s16 = t.s16; + else + assert(0); break; - case S32_TYPE: + case S32_TYPE: t.s64 = ((long long)a.s32) * ((long long)b.s32); - if ( pI->is_wide() ) d.s64 = t.s64; - else if ( pI->is_hi() ) d.s32 = (t.s64>>32); - else if ( pI->is_lo() ) d.s32 = t.s32; - else assert(0); + if (pI->is_wide()) + d.s64 = t.s64; + else if (pI->is_hi()) + d.s32 = (t.s64 >> 32); + else if (pI->is_lo()) + d.s32 = t.s32; + else + assert(0); break; - case S64_TYPE: + case S64_TYPE: t.s64 = a.s64 * b.s64; - assert( !pI->is_wide() ); - assert( !pI->is_hi() ); - if ( pI->is_lo() ) d.s64 = t.s64; - else assert(0); + assert(!pI->is_wide()); + assert(!pI->is_hi()); + if (pI->is_lo()) + d.s64 = t.s64; + else + assert(0); break; - case U16_TYPE: + case U16_TYPE: t.u32 = ((unsigned)a.u16) * ((unsigned)b.u16); - if ( pI->is_wide() ) d.u32 = t.u32; - else if ( pI->is_lo() ) d.u16 = t.u16; - else if ( pI->is_hi() ) d.u16 = (t.u32>>16); - else assert(0); + if (pI->is_wide()) + d.u32 = t.u32; + else if (pI->is_lo()) + d.u16 = t.u16; + else if (pI->is_hi()) + d.u16 = (t.u32 >> 16); + else + assert(0); break; - case U32_TYPE: + case U32_TYPE: t.u64 = ((unsigned long long)a.u32) * ((unsigned long long)b.u32); - if ( pI->is_wide() ) d.u64 = t.u64; - else if ( pI->is_lo() ) d.u32 = t.u32; - else if ( pI->is_hi() ) d.u32 = (t.u64>>32); - else assert(0); + if (pI->is_wide()) + d.u64 = t.u64; + else if (pI->is_lo()) + d.u32 = t.u32; + else if (pI->is_hi()) + d.u32 = (t.u64 >> 32); + else + assert(0); break; - case U64_TYPE: + case U64_TYPE: t.u64 = a.u64 * b.u64; - assert( !pI->is_wide() ); - assert( !pI->is_hi() ); - if ( pI->is_lo() ) d.u64 = t.u64; - else assert(0); - break; - case F16_TYPE:{ - //assert(0); - //break; - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - - d.f16 = a.f16 * b.f16; - - if ( pI->saturation_mode() ) { - if ( d.f16 < 0 ) d.f16 = 0; - else if ( d.f16 > 1.0f ) d.f16 = 1.0f; - } - fesetround( orig_rm ); - break; - } - case F32_TYPE: { - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - - d.f32 = a.f32 * b.f32; - - if ( pI->saturation_mode() ) { - if ( d.f32 < 0 ) d.f32 = 0; - else if ( d.f32 > 1.0f ) d.f32 = 1.0f; - } - fesetround( orig_rm ); - break; - } - case F64_TYPE: case FF64_TYPE:{ - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - d.f64 = a.f64 * b.f64; - if ( pI->saturation_mode() ) { - if ( d.f64 < 0 ) d.f64 = 0; - else if ( d.f64 > 1.0f ) d.f64 = 1.0; - } - fesetround( orig_rm ); - break; - } - default: - assert(0); + assert(!pI->is_wide()); + assert(!pI->is_hi()); + if (pI->is_lo()) + d.u64 = t.u64; + else + assert(0); break; - } + case F16_TYPE: { + // assert(0); + // break; + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } - thread->set_operand_value(dst, d, i_type, thread, pI); -} + d.f16 = a.f16 * b.f16; -void neg_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; + if (pI->saturation_mode()) { + if (d.f16 < 0) + d.f16 = 0; + else if (d.f16 > 1.0f) + d.f16 = 1.0f; + } + fesetround(orig_rm); + break; + } + case F32_TYPE: { + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); + d.f32 = a.f32 * b.f32; - unsigned to_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, to_type, thread, 1); + if (pI->saturation_mode()) { + if (d.f32 < 0) + d.f32 = 0; + else if (d.f32 > 1.0f) + d.f32 = 1.0f; + } + fesetround(orig_rm); + break; + } + case F64_TYPE: + case FF64_TYPE: { + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + d.f64 = a.f64 * b.f64; + if (pI->saturation_mode()) { + if (d.f64 < 0) + d.f64 = 0; + else if (d.f64 > 1.0f) + d.f64 = 1.0; + } + fesetround(orig_rm); + break; + } + default: + assert(0); + break; + } + thread->set_operand_value(dst, d, i_type, thread, pI); +} - switch ( to_type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - data.s64 = 0 - src1_data.s64; break; // seems buggy, but not (just ignore higher bits) - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - assert(0); break; - case F16_TYPE: data.f16 =0.0f - src1_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = 0.0f - src1_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = 0.0f - src1_data.f64; break; - default: assert(0); break; - } +void neg_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned to_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, to_type, thread, 1); + + switch (to_type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + data.s64 = 0 - src1_data.s64; + break; // seems buggy, but not (just ignore higher bits) + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + assert(0); + break; + case F16_TYPE: + data.f16 = 0.0f - src1_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = 0.0f - src1_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = 0.0f - src1_data.f64; + break; + default: + assert(0); + break; + } - thread->set_operand_value(dst,data, to_type, thread, pI); + thread->set_operand_value(dst, data, to_type, thread, pI); } -//nandn bitwise negates second operand then bitwise nands with the first operand -void nandn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +// nandn bitwise negates second operand then bitwise nands with the first +// operand +void nandn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = (~src1_data.pred & src2_data.pred); + else + data.u64 = ~(src1_data.u64 & ~src2_data.u64); - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + thread->set_operand_value(dst, data, i_type, thread, pI); +} - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); +// norn bitwise negates first operand then bitwise ands with the second operand +void norn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = (~src1_data.pred & src2_data.pred); - else - data.u64 = ~(src1_data.u64 & ~src2_data.u64); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - thread->set_operand_value(dst,data, i_type, thread, pI); + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(src1_data.pred & ~(src2_data.pred)); + else + data.u64 = ~(src1_data.u64) & src2_data.u64; + thread->set_operand_value(dst, data, i_type, thread, pI); } -//norn bitwise negates first operand then bitwise ands with the second operand -void norn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +void not_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + switch (i_type) { + case PRED_TYPE: + d.pred = (~(a.pred) & 0x000F); + break; + case B16_TYPE: + d.u16 = ~a.u16; + break; + case B32_TYPE: + d.u32 = ~a.u32; + break; + case B64_TYPE: + d.u64 = ~a.u64; + break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + thread->set_operand_value(dst, d, i_type, thread, pI); +} +void or_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(src1_data.pred & ~(src2_data.pred)); - else - data.u64 = ~(src1_data.u64) & src2_data.u64; + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - thread->set_operand_value(dst,data, i_type, thread, pI); + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) | ~(src2_data.pred)); + else + data.u64 = src1_data.u64 | src2_data.u64; + thread->set_operand_value(dst, data, i_type, thread, pI); } -void not_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void orn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) | (src2_data.pred)); + else + data.u64 = src1_data.u64 | ~src2_data.u64; - switch ( i_type ) { - case PRED_TYPE: d.pred = (~(a.pred) & 0x000F); break; - case B16_TYPE: d.u16 = ~a.u16; break; - case B32_TYPE: d.u32 = ~a.u32; break; - case B64_TYPE: d.u64 = ~a.u64; break; - default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } + thread->set_operand_value(dst, data, i_type, thread, pI); +} - thread->set_operand_value(dst,d, i_type, thread, pI); +void pmevent_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); } +void popc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src = pI->src1(); -void or_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + src_data = thread->get_operand_value(src, dst, i_type, thread, 1); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + switch (i_type) { + case B32_TYPE: { + std::bitset<32> mask(src_data.u32); + data.u32 = mask.count(); + } break; + case B64_TYPE: { + std::bitset<64> mask(src_data.u64); + data.u32 = mask.count(); + } break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) | ~(src2_data.pred)); - else - data.u64 = src1_data.u64 | src2_data.u64; + thread->set_operand_value(dst, data, i_type, thread, pI); +} +void prefetch_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void prefetchu_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} - thread->set_operand_value(dst,data, i_type, thread, pI); +int prmt_mode_present(int mode) { + int returnval = 0; + switch (mode) { + case PRMT_F4E_MODE: + case PRMT_B4E_MODE: + case PRMT_RC8_MODE: + case PRMT_RC16_MODE: + case PRMT_ECL_MODE: + case PRMT_ECR_MODE: + returnval = 1; + break; + default: + break; + } + return returnval; +} +int read_byte(int mode, int control, int d_sel_index, signed long long value) { + int returnval = 0; + int prmt_f4e_mode[4][4] = { + {0, 1, 2, 3}, {1, 2, 3, 4}, {2, 3, 4, 5}, {3, 4, 5, 6}}; + int prmt_b4e_mode[4][4] = { + {0, 7, 6, 5}, {1, 0, 7, 6}, {2, 1, 0, 7}, {3, 2, 1, 0}}; + int prmt_rc8_mode[4][4] = { + {0, 0, 0, 0}, {1, 1, 1, 1}, {2, 2, 2, 2}, {3, 3, 3, 3}}; + int prmt_ecl_mode[4][4] = { + {0, 1, 2, 3}, {1, 1, 2, 3}, {2, 2, 2, 3}, {3, 3, 3, 3}}; + int prmt_ecr_mode[4][4] = { + {0, 0, 0, 0}, {0, 1, 1, 1}, {0, 1, 2, 2}, {0, 1, 2, 3}}; + int prmt_rc16_mode[4][4] = { + {0, 1, 0, 1}, {2, 3, 2, 3}, {0, 1, 0, 1}, {2, 3, 2, 3}}; + + if (!prmt_mode_present(mode)) { + if (control & 0x8) { + returnval = 0xff; + } else { + returnval = (value >> (8 * control)) & 0xff; + } + } else { + switch (mode) { + case PRMT_F4E_MODE: + returnval = prmt_f4e_mode[control][d_sel_index]; + break; + case PRMT_B4E_MODE: + returnval = prmt_b4e_mode[control][d_sel_index]; + break; + case PRMT_RC8_MODE: + returnval = prmt_rc8_mode[control][d_sel_index]; + break; + case PRMT_ECL_MODE: + returnval = prmt_ecl_mode[control][d_sel_index]; + break; + case PRMT_ECR_MODE: + returnval = prmt_ecr_mode[control][d_sel_index]; + break; + case PRMT_RC16_MODE: + returnval = prmt_rc16_mode[control][d_sel_index]; + break; + // Change the default from printing "ERROR" to just asserting + default: + assert(false); + } + } + return (returnval << 8 * d_sel_index); } -void orn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) | (src2_data.pred)); - else - data.u64 = src1_data.u64 | ~src2_data.u64; +void prmt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, src3_data, tmpdata, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + unsigned mode = pI->prmt_op(); + unsigned i_type = pI->get_type(); + + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + + tmpdata.s64 = src1_data.s32 | (src2_data.s64 << 32); + int ctl[4]; + + if (!prmt_mode_present(mode)) { + ctl[0] = (src3_data.s32 >> 0) & 0xf; + ctl[1] = (src3_data.s32 >> 4) & 0xf; + ctl[2] = (src3_data.s32 >> 8) & 0xf; + ctl[3] = (src3_data.s32 >> 12) & 0xf; + } else { + ctl[0] = ctl[1] = ctl[2] = ctl[3] = (src3_data.s32 >> 0) & 0x3; + } - thread->set_operand_value(dst,data, i_type, thread, pI); + data.s32 = 0; + data.s32 = data.s32 | read_byte(mode, ctl[0], 0, tmpdata.s64); // First + // byte-0 + data.s32 = + data.s32 | read_byte(mode, ctl[1], 1, tmpdata.s64); // Second byte-1 + data.s32 = data.s32 | read_byte(mode, ctl[2], 2, tmpdata.s64); // Third + // byte-2 + data.s32 = + data.s32 | read_byte(mode, ctl[3], 3, tmpdata.s64); // Fourth byte-3 + + thread->set_operand_value(dst, data, i_type, thread, pI); } -void pmevent_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void popc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src = pI->src1(); +void rcp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - src_data = thread->get_operand_value(src, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch ( i_type ) { - case B32_TYPE: { - std::bitset<32> mask(src_data.u32); - data.u32 = mask.count(); - } break; - case B64_TYPE: { - std::bitset<64> mask(src_data.u64); - data.u32 = mask.count(); - } break; - default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - - thread->set_operand_value(dst,data, i_type, thread, pI); -} -void prefetch_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void prefetchu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } - -int prmt_mode_present(int mode) -{ - int returnval=0; - switch(mode){ - case PRMT_F4E_MODE: - case PRMT_B4E_MODE: - case PRMT_RC8_MODE: - case PRMT_RC16_MODE: - case PRMT_ECL_MODE: - case PRMT_ECR_MODE: - returnval=1; - break; - default: - break; - } - return returnval; -} -int read_byte(int mode, int control, int d_sel_index, signed long long value){ - - int returnval = 0; - int prmt_f4e_mode[4][4]={{0,1,2,3},{1,2,3,4},{2,3,4,5},{3,4,5,6}}; - int prmt_b4e_mode[4][4]={{0,7,6,5},{1,0,7,6},{2,1,0,7},{3,2,1,0}}; - int prmt_rc8_mode[4][4]={{0,0,0,0},{1,1,1,1},{2,2,2,2},{3,3,3,3}}; - int prmt_ecl_mode[4][4]={{0,1,2,3},{1,1,2,3},{2,2,2,3},{3,3,3,3}}; - int prmt_ecr_mode[4][4]={{0,0,0,0},{0,1,1,1},{0,1,2,2},{0,1,2,3}}; - int prmt_rc16_mode[4][4]={{0,1,0,1},{2,3,2,3},{0,1,0,1},{2,3,2,3}}; - - if(!prmt_mode_present(mode)){ - if(control&0x8){ - returnval=0xff; - } - else{ - returnval= (value>>(8*control)) & 0xff; - } - } - else{ - switch(mode){ - case PRMT_F4E_MODE: returnval=prmt_f4e_mode[control][d_sel_index];break; - case PRMT_B4E_MODE: returnval=prmt_b4e_mode[control][d_sel_index];break; - case PRMT_RC8_MODE: returnval=prmt_rc8_mode[control][d_sel_index];break; - case PRMT_ECL_MODE: returnval=prmt_ecl_mode[control][d_sel_index];break; - case PRMT_ECR_MODE: returnval=prmt_ecr_mode[control][d_sel_index];break; - case PRMT_RC16_MODE: returnval=prmt_rc16_mode[control][d_sel_index];break; - // Change the default from printing "ERROR" to just asserting - default: assert(false); - } - } - return (returnval << 8 * d_sel_index); -} - -void prmt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { - - ptx_reg_t src1_data, src2_data, src3_data,tmpdata,data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - - unsigned mode = pI->prmt_op(); - unsigned i_type = pI->get_type(); - - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - - tmpdata.s64=src1_data.s32|(src2_data.s64<<32); - int ctl[4]; - - if(!prmt_mode_present(mode)){ - ctl[0]=(src3_data.s32>>0)&0xf; - ctl[1]=(src3_data.s32>>4)&0xf; - ctl[2]=(src3_data.s32>>8)&0xf; - ctl[3]=(src3_data.s32>>12)&0xf; - } - else{ - ctl[0]=ctl[1]=ctl[2]=ctl[3]=(src3_data.s32>>0)&0x3; - } - - data.s32=0; - data.s32=data.s32|read_byte(mode,ctl[0],0,tmpdata.s64); //First byte-0 - data.s32=data.s32|read_byte(mode,ctl[1],1,tmpdata.s64); //Second byte-1 - data.s32=data.s32|read_byte(mode,ctl[2],2,tmpdata.s64); //Third byte-2 - data.s32=data.s32|read_byte(mode,ctl[3],3,tmpdata.s64); //Fourth byte-3 - - thread->set_operand_value(dst,data, i_type, thread, pI); - - -} - -void rcp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case F32_TYPE: + switch (i_type) { + case F32_TYPE: data.f32 = 1.0f / src1_data.f32; break; - case F64_TYPE: - case FF64_TYPE: + case F64_TYPE: + case FF64_TYPE: data.f64 = 1.0f / src1_data.f64; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void red_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void red_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} -void rem_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +void rem_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case S32_TYPE: + switch (i_type) { + case S32_TYPE: data.s32 = src1_data.s32 % src2_data.s32; break; - case S64_TYPE: + case S64_TYPE: data.s64 = src1_data.s64 % src2_data.s64; break; - case U32_TYPE: + case U32_TYPE: data.u32 = src1_data.u32 % src2_data.u32; break; - case U64_TYPE: + case U64_TYPE: data.u64 = src1_data.u64 % src2_data.u64; break; - default: assert(0); break; - } + default: + assert(0); + break; + } - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void ret_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - bool empty = thread->callstack_pop(); - if( empty ) { - thread->set_done(); - thread->exitCore(); - thread->registerExit(); - } +void ret_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + bool empty = thread->callstack_pop(); + if (empty) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); + } } -//Ptxplus version of ret instruction. -void retp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - bool empty = thread->callstack_pop_plus(); - if( empty ) { - thread->set_done(); - thread->exitCore(); - thread->registerExit(); - } +// Ptxplus version of ret instruction. +void retp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + bool empty = thread->callstack_pop_plus(); + if (empty) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); + } } -void rsqrt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case F32_TYPE: - if ( a.f32 < 0 ) { - d.u64 = 0; - d.u64 = 0x7fc00000; // NaN - } else if ( a.f32 == 0 ) { - d.u64 = 0; - d.u32 = 0x7f800000; // Inf +void rsqrt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch (i_type) { + case F32_TYPE: + if (a.f32 < 0) { + d.u64 = 0; + d.u64 = 0x7fc00000; // NaN + } else if (a.f32 == 0) { + d.u64 = 0; + d.u32 = 0x7f800000; // Inf } else - d.f32 = cuda_math::__internal_accurate_fdividef(1.0f, sqrtf(a.f32)); - break; - case F64_TYPE: - case FF64_TYPE: - if ( a.f32 < 0 ) { - d.u64 = 0; - d.u32 = 0x7fc00000; // NaN - float x = d.f32; - d.f64 = (double)x; - } else if ( a.f32 == 0 ) { - d.u64 = 0; - d.u32 = 0x7f800000; // Inf - float x = d.f32; - d.f64 = (double)x; + d.f32 = cuda_math::__internal_accurate_fdividef(1.0f, sqrtf(a.f32)); + break; + case F64_TYPE: + case FF64_TYPE: + if (a.f32 < 0) { + d.u64 = 0; + d.u32 = 0x7fc00000; // NaN + float x = d.f32; + d.f64 = (double)x; + } else if (a.f32 == 0) { + d.u64 = 0; + d.u32 = 0x7f800000; // Inf + float x = d.f32; + d.f64 = (double)x; } else - d.f64 = 1.0 / sqrt(a.f64); + d.f64 = 1.0 / sqrt(a.f64); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -#define SAD(d,a,b,c) d = c + ((adst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); +void sad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, c, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, i_type, thread, 1); - - switch ( i_type ) { - case U16_TYPE: SAD(d.u16,a.u16,b.u16,c.u16); break; - case U32_TYPE: SAD(d.u32,a.u32,b.u32,c.u32); break; - case U64_TYPE: SAD(d.u64,a.u64,b.u64,c.u64); break; - case S16_TYPE: SAD(d.s16,a.s16,b.s16,c.s16); break; - case S32_TYPE: SAD(d.s32,a.s32,b.s32,c.s32); break; - case S64_TYPE: SAD(d.s64,a.s64,b.s64,c.s64); break; - case F32_TYPE: SAD(d.f32,a.f32,b.f32,c.f32); break; - case F64_TYPE: case FF64_TYPE: SAD(d.f64,a.f64,b.f64,c.f64); break; - default: + switch (i_type) { + case U16_TYPE: + SAD(d.u16, a.u16, b.u16, c.u16); + break; + case U32_TYPE: + SAD(d.u32, a.u32, b.u32, c.u32); + break; + case U64_TYPE: + SAD(d.u64, a.u64, b.u64, c.u64); + break; + case S16_TYPE: + SAD(d.s16, a.s16, b.s16, c.s16); + break; + case S32_TYPE: + SAD(d.s32, a.s32, b.s32, c.s32); + break; + case S64_TYPE: + SAD(d.s64, a.s64, b.s64, c.s64); + break; + case F32_TYPE: + SAD(d.f32, a.f32, b.f32, c.f32); + break; + case F64_TYPE: + case FF64_TYPE: + SAD(d.f64, a.f64, b.f64, c.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void selp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); +void selp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); - ptx_reg_t a, b, c, d; + ptx_reg_t a, b, c, d; - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, i_type, thread, 1); - //predicate value was changed so the lowest bit being set means the zero flag is set. - //As a result, the value of c.pred must be inverted to get proper behavior - d = (!(c.pred & 0x0001))?a:b; + // predicate value was changed so the lowest bit being set means the zero flag + // is set. + // As a result, the value of c.pred must be inverted to get proper behavior + d = (!(c.pred & 0x0001)) ? a : b; - thread->set_operand_value(dst,d, PRED_TYPE, thread, pI); + thread->set_operand_value(dst, d, PRED_TYPE, thread, pI); } -bool isFloat(int type) -{ - switch ( type ) { - case F16_TYPE: - case F32_TYPE: - case F64_TYPE: - case FF64_TYPE: +bool isFloat(int type) { + switch (type) { + case F16_TYPE: + case F32_TYPE: + case F64_TYPE: + case FF64_TYPE: return true; - default: + default: return false; - } + } } -bool CmpOp( int type, ptx_reg_t a, ptx_reg_t b, unsigned cmpop ) -{ - bool t = false; +bool CmpOp(int type, ptx_reg_t a, ptx_reg_t b, unsigned cmpop) { + bool t = false; - switch ( type ) { - case B16_TYPE: + switch (type) { + case B16_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u16 == b.u16); break; - case NE_OPTION: t = (a.u16 != b.u16); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u16 == b.u16); + break; + case NE_OPTION: + t = (a.u16 != b.u16); + break; + default: + assert(0); } - case B32_TYPE: + case B32_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u32 == b.u32); break; - case NE_OPTION: t = (a.u32 != b.u32); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u32 == b.u32); + break; + case NE_OPTION: + t = (a.u32 != b.u32); + break; + default: + assert(0); } - case B64_TYPE: + case B64_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u64 == b.u64); break; - case NE_OPTION: t = (a.u64 != b.u64); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u64 == b.u64); + break; + case NE_OPTION: + t = (a.u64 != b.u64); + break; + default: + assert(0); } break; - case S8_TYPE: - case S16_TYPE: + case S8_TYPE: + case S16_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.s16 == b.s16); break; - case NE_OPTION: t = (a.s16 != b.s16); break; - case LT_OPTION: t = (a.s16 < b.s16); break; - case LE_OPTION: t = (a.s16 <= b.s16); break; - case GT_OPTION: t = (a.s16 > b.s16); break; - case GE_OPTION: t = (a.s16 >= b.s16); break; - default: - assert(0); + case EQ_OPTION: + t = (a.s16 == b.s16); + break; + case NE_OPTION: + t = (a.s16 != b.s16); + break; + case LT_OPTION: + t = (a.s16 < b.s16); + break; + case LE_OPTION: + t = (a.s16 <= b.s16); + break; + case GT_OPTION: + t = (a.s16 > b.s16); + break; + case GE_OPTION: + t = (a.s16 >= b.s16); + break; + default: + assert(0); } break; - case S32_TYPE: + case S32_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.s32 == b.s32); break; - case NE_OPTION: t = (a.s32 != b.s32); break; - case LT_OPTION: t = (a.s32 < b.s32); break; - case LE_OPTION: t = (a.s32 <= b.s32); break; - case GT_OPTION: t = (a.s32 > b.s32); break; - case GE_OPTION: t = (a.s32 >= b.s32); break; - default: - assert(0); + case EQ_OPTION: + t = (a.s32 == b.s32); + break; + case NE_OPTION: + t = (a.s32 != b.s32); + break; + case LT_OPTION: + t = (a.s32 < b.s32); + break; + case LE_OPTION: + t = (a.s32 <= b.s32); + break; + case GT_OPTION: + t = (a.s32 > b.s32); + break; + case GE_OPTION: + t = (a.s32 >= b.s32); + break; + default: + assert(0); } break; - case S64_TYPE: + case S64_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.s64 == b.s64); break; - case NE_OPTION: t = (a.s64 != b.s64); break; - case LT_OPTION: t = (a.s64 < b.s64); break; - case LE_OPTION: t = (a.s64 <= b.s64); break; - case GT_OPTION: t = (a.s64 > b.s64); break; - case GE_OPTION: t = (a.s64 >= b.s64); break; - default: - assert(0); + case EQ_OPTION: + t = (a.s64 == b.s64); + break; + case NE_OPTION: + t = (a.s64 != b.s64); + break; + case LT_OPTION: + t = (a.s64 < b.s64); + break; + case LE_OPTION: + t = (a.s64 <= b.s64); + break; + case GT_OPTION: + t = (a.s64 > b.s64); + break; + case GE_OPTION: + t = (a.s64 >= b.s64); + break; + default: + assert(0); } break; - case U8_TYPE: - case U16_TYPE: + case U8_TYPE: + case U16_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u16 == b.u16); break; - case NE_OPTION: t = (a.u16 != b.u16); break; - case LT_OPTION: t = (a.u16 < b.u16); break; - case LE_OPTION: t = (a.u16 <= b.u16); break; - case GT_OPTION: t = (a.u16 > b.u16); break; - case GE_OPTION: t = (a.u16 >= b.u16); break; - case LO_OPTION: t = (a.u16 < b.u16); break; - case LS_OPTION: t = (a.u16 <= b.u16); break; - case HI_OPTION: t = (a.u16 > b.u16); break; - case HS_OPTION: t = (a.u16 >= b.u16); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u16 == b.u16); + break; + case NE_OPTION: + t = (a.u16 != b.u16); + break; + case LT_OPTION: + t = (a.u16 < b.u16); + break; + case LE_OPTION: + t = (a.u16 <= b.u16); + break; + case GT_OPTION: + t = (a.u16 > b.u16); + break; + case GE_OPTION: + t = (a.u16 >= b.u16); + break; + case LO_OPTION: + t = (a.u16 < b.u16); + break; + case LS_OPTION: + t = (a.u16 <= b.u16); + break; + case HI_OPTION: + t = (a.u16 > b.u16); + break; + case HS_OPTION: + t = (a.u16 >= b.u16); + break; + default: + assert(0); } break; - case U32_TYPE: + case U32_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u32 == b.u32); break; - case NE_OPTION: t = (a.u32 != b.u32); break; - case LT_OPTION: t = (a.u32 < b.u32); break; - case LE_OPTION: t = (a.u32 <= b.u32); break; - case GT_OPTION: t = (a.u32 > b.u32); break; - case GE_OPTION: t = (a.u32 >= b.u32); break; - case LO_OPTION: t = (a.u32 < b.u32); break; - case LS_OPTION: t = (a.u32 <= b.u32); break; - case HI_OPTION: t = (a.u32 > b.u32); break; - case HS_OPTION: t = (a.u32 >= b.u32); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u32 == b.u32); + break; + case NE_OPTION: + t = (a.u32 != b.u32); + break; + case LT_OPTION: + t = (a.u32 < b.u32); + break; + case LE_OPTION: + t = (a.u32 <= b.u32); + break; + case GT_OPTION: + t = (a.u32 > b.u32); + break; + case GE_OPTION: + t = (a.u32 >= b.u32); + break; + case LO_OPTION: + t = (a.u32 < b.u32); + break; + case LS_OPTION: + t = (a.u32 <= b.u32); + break; + case HI_OPTION: + t = (a.u32 > b.u32); + break; + case HS_OPTION: + t = (a.u32 >= b.u32); + break; + default: + assert(0); } break; - case U64_TYPE: + case U64_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u64 == b.u64); break; - case NE_OPTION: t = (a.u64 != b.u64); break; - case LT_OPTION: t = (a.u64 < b.u64); break; - case LE_OPTION: t = (a.u64 <= b.u64); break; - case GT_OPTION: t = (a.u64 > b.u64); break; - case GE_OPTION: t = (a.u64 >= b.u64); break; - case LO_OPTION: t = (a.u64 < b.u64); break; - case LS_OPTION: t = (a.u64 <= b.u64); break; - case HI_OPTION: t = (a.u64 > b.u64); break; - case HS_OPTION: t = (a.u64 >= b.u64); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u64 == b.u64); + break; + case NE_OPTION: + t = (a.u64 != b.u64); + break; + case LT_OPTION: + t = (a.u64 < b.u64); + break; + case LE_OPTION: + t = (a.u64 <= b.u64); + break; + case GT_OPTION: + t = (a.u64 > b.u64); + break; + case GE_OPTION: + t = (a.u64 >= b.u64); + break; + case LO_OPTION: + t = (a.u64 < b.u64); + break; + case LS_OPTION: + t = (a.u64 <= b.u64); + break; + case HI_OPTION: + t = (a.u64 > b.u64); + break; + case HS_OPTION: + t = (a.u64 >= b.u64); + break; + default: + assert(0); } break; - case F16_TYPE: assert(0); break; - case F32_TYPE: + case F16_TYPE: + assert(0); + break; + case F32_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.f32 == b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; - case NE_OPTION: t = (a.f32 != b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; - case LT_OPTION: t = (a.f32 < b.f32 ) && !isNaN(a.f32) && !isNaN(b.f32); break; - case LE_OPTION: t = (a.f32 <= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; - case GT_OPTION: t = (a.f32 > b.f32 ) && !isNaN(a.f32) && !isNaN(b.f32); break; - case GE_OPTION: t = (a.f32 >= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; - case EQU_OPTION: t = (a.f32 == b.f32) || isNaN(a.f32) || isNaN(b.f32); break; - case NEU_OPTION: t = (a.f32 != b.f32) || isNaN(a.f32) || isNaN(b.f32); break; - case LTU_OPTION: t = (a.f32 < b.f32 ) || isNaN(a.f32) || isNaN(b.f32); break; - case LEU_OPTION: t = (a.f32 <= b.f32) || isNaN(a.f32) || isNaN(b.f32); break; - case GTU_OPTION: t = (a.f32 > b.f32 ) || isNaN(a.f32) || isNaN(b.f32); break; - case GEU_OPTION: t = (a.f32 >= b.f32) || isNaN(a.f32) || isNaN(b.f32); break; - case NUM_OPTION: t = !isNaN(a.f32) && !isNaN(b.f32); break; - case NAN_OPTION: t = isNaN(a.f32) || isNaN(b.f32); break; - default: - assert(0); + case EQ_OPTION: + t = (a.f32 == b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case NE_OPTION: + t = (a.f32 != b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case LT_OPTION: + t = (a.f32 < b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case LE_OPTION: + t = (a.f32 <= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case GT_OPTION: + t = (a.f32 > b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case GE_OPTION: + t = (a.f32 >= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case EQU_OPTION: + t = (a.f32 == b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case NEU_OPTION: + t = (a.f32 != b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case LTU_OPTION: + t = (a.f32 < b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case LEU_OPTION: + t = (a.f32 <= b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case GTU_OPTION: + t = (a.f32 > b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case GEU_OPTION: + t = (a.f32 >= b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case NUM_OPTION: + t = !isNaN(a.f32) && !isNaN(b.f32); + break; + case NAN_OPTION: + t = isNaN(a.f32) || isNaN(b.f32); + break; + default: + assert(0); } break; - case F64_TYPE: - case FF64_TYPE: + case F64_TYPE: + case FF64_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.f64 == b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; - case NE_OPTION: t = (a.f64 != b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; - case LT_OPTION: t = (a.f64 < b.f64 ) && !isNaN(a.f64) && !isNaN(b.f64); break; - case LE_OPTION: t = (a.f64 <= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; - case GT_OPTION: t = (a.f64 > b.f64 ) && !isNaN(a.f64) && !isNaN(b.f64); break; - case GE_OPTION: t = (a.f64 >= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; - case EQU_OPTION: t = (a.f64 == b.f64) || isNaN(a.f64) || isNaN(b.f64); break; - case NEU_OPTION: t = (a.f64 != b.f64) || isNaN(a.f64) || isNaN(b.f64); break; - case LTU_OPTION: t = (a.f64 < b.f64 ) || isNaN(a.f64) || isNaN(b.f64); break; - case LEU_OPTION: t = (a.f64 <= b.f64) || isNaN(a.f64) || isNaN(b.f64); break; - case GTU_OPTION: t = (a.f64 > b.f64 ) || isNaN(a.f64) || isNaN(b.f64); break; - case GEU_OPTION: t = (a.f64 >= b.f64) || isNaN(a.f64) || isNaN(b.f64); break; - case NUM_OPTION: t = !isNaN(a.f64) && !isNaN(b.f64); break; - case NAN_OPTION: t = isNaN(a.f64) || isNaN(b.f64); break; - default: - assert(0); + case EQ_OPTION: + t = (a.f64 == b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case NE_OPTION: + t = (a.f64 != b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case LT_OPTION: + t = (a.f64 < b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case LE_OPTION: + t = (a.f64 <= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case GT_OPTION: + t = (a.f64 > b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case GE_OPTION: + t = (a.f64 >= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case EQU_OPTION: + t = (a.f64 == b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case NEU_OPTION: + t = (a.f64 != b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case LTU_OPTION: + t = (a.f64 < b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case LEU_OPTION: + t = (a.f64 <= b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case GTU_OPTION: + t = (a.f64 > b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case GEU_OPTION: + t = (a.f64 >= b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case NUM_OPTION: + t = !isNaN(a.f64) && !isNaN(b.f64); + break; + case NAN_OPTION: + t = isNaN(a.f64) || isNaN(b.f64); + break; + default: + assert(0); } break; - default: assert(0); break; - } + default: + assert(0); + break; + } - return t; + return t; } -void setp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b; +void setp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b; - int t=0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + int t = 0; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - assert( pI->get_num_operands() < 4 ); // or need to deal with "c" operand / boolOp + assert(pI->get_num_operands() < + 4); // or need to deal with "c" operand / boolOp - unsigned type = pI->get_type(); - unsigned cmpop = pI->get_cmpop(); - a = thread->get_operand_value(src1, dst, type, thread, 1); - b = thread->get_operand_value(src2, dst, type, thread, 1); + unsigned type = pI->get_type(); + unsigned cmpop = pI->get_cmpop(); + a = thread->get_operand_value(src1, dst, type, thread, 1); + b = thread->get_operand_value(src2, dst, type, thread, 1); - t = CmpOp(type,a,b,cmpop); + t = CmpOp(type, a, b, cmpop); - ptx_reg_t data; + ptx_reg_t data; - //the way ptxplus handles the zero flag, 1 = false and 0 = true - data.pred = (t==0); //inverting predicate since ptxplus uses "1" for a set zero flag + // the way ptxplus handles the zero flag, 1 = false and 0 = true + data.pred = + (t == + 0); // inverting predicate since ptxplus uses "1" for a set zero flag - thread->set_operand_value(dst,data, PRED_TYPE, thread, pI); + thread->set_operand_value(dst, data, PRED_TYPE, thread, pI); } -void set_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b; +void set_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b; - int t=0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + int t = 0; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - assert( pI->get_num_operands() < 4 ); // or need to deal with "c" operand / boolOp + assert(pI->get_num_operands() < + 4); // or need to deal with "c" operand / boolOp - unsigned src_type = pI->get_type2(); - unsigned cmpop = pI->get_cmpop(); + unsigned src_type = pI->get_type2(); + unsigned cmpop = pI->get_cmpop(); - a = thread->get_operand_value(src1, dst, src_type, thread, 1); - b = thread->get_operand_value(src2, dst, src_type, thread, 1); + a = thread->get_operand_value(src1, dst, src_type, thread, 1); + b = thread->get_operand_value(src2, dst, src_type, thread, 1); - // Take abs of first operand if needed - if(pI->is_abs()) { - switch ( src_type ) { - case S16_TYPE: a.s16 = my_abs(a.s16); break; - case S32_TYPE: a.s32 = my_abs(a.s32); break; - case S64_TYPE: a.s64 = my_abs(a.s64); break; - case U16_TYPE: a.u16 = a.u16; break; - case U32_TYPE: a.u32 = my_abs(a.u32); break; - case U64_TYPE: a.u64 = my_abs(a.u64); break; - case F32_TYPE: a.f32 = my_abs(a.f32); break; - case F64_TYPE: case FF64_TYPE: a.f64 = my_abs(a.f64); break; + // Take abs of first operand if needed + if (pI->is_abs()) { + switch (src_type) { + case S16_TYPE: + a.s16 = my_abs(a.s16); + break; + case S32_TYPE: + a.s32 = my_abs(a.s32); + break; + case S64_TYPE: + a.s64 = my_abs(a.s64); + break; + case U16_TYPE: + a.u16 = a.u16; + break; + case U32_TYPE: + a.u32 = my_abs(a.u32); + break; + case U64_TYPE: + a.u64 = my_abs(a.u64); + break; + case F32_TYPE: + a.f32 = my_abs(a.f32); + break; + case F64_TYPE: + case FF64_TYPE: + a.f64 = my_abs(a.f64); + break; default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - } - - t = CmpOp(src_type,a,b,cmpop); - - ptx_reg_t data; - if ( isFloat(pI->get_type()) ) { - data.f32 = (t!=0)?1.0f:0.0f; - } else { - data.u32 = (t!=0)?0xFFFFFFFF:0; - } - - thread->set_operand_value(dst, data, pI->get_type(), thread, pI); - -} - -void shfl_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) -{ - unsigned i_type = pI->get_type(); - int tid; - - if(core->get_gpu()->is_functional_sim()) - tid = inst.warp_id_func() * core->get_warp_size(); - else - tid = inst.warp_id() * core->get_warp_size(); - - ptx_thread_info *thread = core->get_thread_info()[tid]; - ptx_warp_info *warp_info = thread->m_warp_info; - int lane = warp_info->get_done_threads(); - thread = core->get_thread_info()[tid + lane]; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - int bval = (thread->get_operand_value(src2, dst, i_type, thread, 1)).u32; - int cval = (thread->get_operand_value(src3, dst, i_type, thread, 1)).u32; - int mask = cval >> 8; - bval &= 0x1F; - cval &= 0x1F; - - int maxLane = (lane & mask) | (cval & ~mask); - int minLane = lane & mask; - - int src_idx; - unsigned p; - switch(pI->shfl_op()) { - case UP_OPTION: - src_idx = lane - bval; - p = (src_idx >= maxLane); - break; - case DOWN_OPTION: - src_idx = lane + bval; - p = (src_idx <= maxLane); - break; - case BFLY_OPTION: - src_idx = lane ^ bval; - p = (src_idx <= maxLane); - break; - case IDX_OPTION: - src_idx = minLane | (bval & ~mask); - p = (src_idx <= maxLane); - break; - default: - printf("GPGPU-Sim PTX: ERROR: Invalid shfl option\n"); - assert(0); - break; - } - // copy from own lane - if (!p) src_idx = lane; - - // copy input from lane src_idx - ptx_reg_t data; - if (inst.active(src_idx)) { - ptx_thread_info *source = core->get_thread_info()[tid + src_idx]; - data = source->get_operand_value(src1, dst, i_type, source, 1); - } else { - printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for inactive threads in a warp\n"); - data.u32 = 0; - } - thread->set_operand_value(dst, data, i_type, thread, pI); - - /* - TODO: deal with predicates appropriately using the following pseudocode: - if (!isGuardPredicateTrue(src_idx)) { - printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for predicated-off threads in a warp\n"); - } - if (dest predicate selected) data.pred = p; - */ - - // keep track of the number of threads that have executed in the warp - warp_info->inc_done_threads(); - if (warp_info->get_done_threads() == inst.active_count()) { - warp_info->reset_done_threads(); - } -} - -void shl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - - switch ( i_type ) { - case B16_TYPE: - case U16_TYPE: - if ( b.u16 >= 16 ) - d.u16 = 0; + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + } + + t = CmpOp(src_type, a, b, cmpop); + + ptx_reg_t data; + if (isFloat(pI->get_type())) { + data.f32 = (t != 0) ? 1.0f : 0.0f; + } else { + data.u32 = (t != 0) ? 0xFFFFFFFF : 0; + } + + thread->set_operand_value(dst, data, pI->get_type(), thread, pI); +} + +void shfl_impl(const ptx_instruction *pI, core_t *core, warp_inst_t inst) { + unsigned i_type = pI->get_type(); + int tid; + + if (core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); + + ptx_thread_info *thread = core->get_thread_info()[tid]; + ptx_warp_info *warp_info = thread->m_warp_info; + int lane = warp_info->get_done_threads(); + thread = core->get_thread_info()[tid + lane]; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + int bval = (thread->get_operand_value(src2, dst, i_type, thread, 1)).u32; + int cval = (thread->get_operand_value(src3, dst, i_type, thread, 1)).u32; + int mask = cval >> 8; + bval &= 0x1F; + cval &= 0x1F; + + int maxLane = (lane & mask) | (cval & ~mask); + int minLane = lane & mask; + + int src_idx; + unsigned p; + switch (pI->shfl_op()) { + case UP_OPTION: + src_idx = lane - bval; + p = (src_idx >= maxLane); + break; + case DOWN_OPTION: + src_idx = lane + bval; + p = (src_idx <= maxLane); + break; + case BFLY_OPTION: + src_idx = lane ^ bval; + p = (src_idx <= maxLane); + break; + case IDX_OPTION: + src_idx = minLane | (bval & ~mask); + p = (src_idx <= maxLane); + break; + default: + printf("GPGPU-Sim PTX: ERROR: Invalid shfl option\n"); + assert(0); + break; + } + // copy from own lane + if (!p) src_idx = lane; + + // copy input from lane src_idx + ptx_reg_t data; + if (inst.active(src_idx)) { + ptx_thread_info *source = core->get_thread_info()[tid + src_idx]; + data = source->get_operand_value(src1, dst, i_type, source, 1); + } else { + printf( + "GPGPU-Sim PTX: WARNING: shfl input value unpredictable for inactive " + "threads in a warp\n"); + data.u32 = 0; + } + thread->set_operand_value(dst, data, i_type, thread, pI); + + /* + TODO: deal with predicates appropriately using the following pseudocode: + if (!isGuardPredicateTrue(src_idx)) { + printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for + predicated-off threads in a warp\n"); + } + if (dest predicate selected) data.pred = p; + */ + + // keep track of the number of threads that have executed in the warp + warp_info->inc_done_threads(); + if (warp_info->get_done_threads() == inst.active_count()) { + warp_info->reset_done_threads(); + } +} + +void shl_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + + switch (i_type) { + case B16_TYPE: + case U16_TYPE: + if (b.u16 >= 16) + d.u16 = 0; else - d.u16 = (unsigned short) ((a.u16 << b.u16) & 0xFFFF); + d.u16 = (unsigned short)((a.u16 << b.u16) & 0xFFFF); break; - case B32_TYPE: - case U32_TYPE: - if ( b.u32 >= 32 ) - d.u32 = 0; + case B32_TYPE: + case U32_TYPE: + if (b.u32 >= 32) + d.u32 = 0; else - d.u32 = (unsigned) ((a.u32 << b.u32) & 0xFFFFFFFF); + d.u32 = (unsigned)((a.u32 << b.u32) & 0xFFFFFFFF); break; - case B64_TYPE: - case U64_TYPE: - if ( b.u32 >= 64 ) - d.u64 = 0; + case B64_TYPE: + case U64_TYPE: + if (b.u32 >= 64) + d.u64 = 0; else - d.u64 = (a.u64 << b.u64); + d.u64 = (a.u64 << b.u64); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void shr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - - - switch ( i_type ) { - case U16_TYPE: - case B16_TYPE: - if ( b.u16 < 16 ) - d.u16 = (unsigned short) ((a.u16 >> b.u16) & 0xFFFF); +void shr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + + switch (i_type) { + case U16_TYPE: + case B16_TYPE: + if (b.u16 < 16) + d.u16 = (unsigned short)((a.u16 >> b.u16) & 0xFFFF); else - d.u16 = 0; + d.u16 = 0; break; - case U32_TYPE: - case B32_TYPE: - if ( b.u32 < 32 ) - d.u32 = (unsigned) ((a.u32 >> b.u32) & 0xFFFFFFFF); + case U32_TYPE: + case B32_TYPE: + if (b.u32 < 32) + d.u32 = (unsigned)((a.u32 >> b.u32) & 0xFFFFFFFF); else - d.u32 = 0; + d.u32 = 0; break; - case U64_TYPE: - case B64_TYPE: - if ( b.u32 < 64 ) - d.u64 = (a.u64 >> b.u64); + case U64_TYPE: + case B64_TYPE: + if (b.u32 < 64) + d.u64 = (a.u64 >> b.u64); else - d.u64 = 0; + d.u64 = 0; break; - case S16_TYPE: - if ( b.u16 < 16 ) - d.s64 = (a.s16 >> b.s16); + case S16_TYPE: + if (b.u16 < 16) + d.s64 = (a.s16 >> b.s16); else { - if ( a.s16 < 0 ) { - d.s64 = -1; - } else { - d.s64 = 0; - } + if (a.s16 < 0) { + d.s64 = -1; + } else { + d.s64 = 0; + } } break; - case S32_TYPE: - if ( b.u32 < 32 ) - d.s64 = (a.s32 >> b.s32); + case S32_TYPE: + if (b.u32 < 32) + d.s64 = (a.s32 >> b.s32); else { - if ( a.s32 < 0 ) { - d.s64 = -1; - } else { - d.s64 = 0; - } + if (a.s32 < 0) { + d.s64 = -1; + } else { + d.s64 = 0; + } } break; - case S64_TYPE: - if ( b.u64 < 64 ) - d.s64 = (a.s64 >> b.u64); + case S64_TYPE: + if (b.u64 < 64) + d.s64 = (a.s64 >> b.u64); else { - if ( a.s64 < 0 ) { - if ( b.s32 < 0 ) { - d.u64 = -1; - d.s32 = 0; - } else { - d.s64 = -1; - } - } else { - d.s64 = 0; - } + if (a.s64 < 0) { + if (b.s32 < 0) { + d.u64 = -1; + d.s32 = 0; + } else { + d.s64 = -1; + } + } else { + d.s64 = 0; + } } break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void sin_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void sin_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - switch ( i_type ) { - case F32_TYPE: + switch (i_type) { + case F32_TYPE: d.f32 = sin(a.f32); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - - thread->set_operand_value(dst,d, i_type, thread, pI); -} - -void slct_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - - ptx_reg_t a, b, c, d; - - unsigned i_type = pI->get_type(); - unsigned c_type = pI->get_type2(); - bool t = false; - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, c_type, thread, 1); - - switch ( c_type ) { - case S32_TYPE: t = c.s32 >= 0; break; - case F32_TYPE: t = c.f32 >= 0; break; - default: assert(0); - } - - switch ( i_type ) { - case B16_TYPE: - case S16_TYPE: - case U16_TYPE: d.u16 = t?a.u16:b.u16; break; - case F32_TYPE: - case B32_TYPE: - case S32_TYPE: - case U32_TYPE: d.u32 = t?a.u32:b.u32; break; - case F64_TYPE: - case FF64_TYPE: - case B64_TYPE: - case S64_TYPE: - case U64_TYPE: d.u64 = t?a.u64:b.u64; break; - default: assert(0); - } - - thread->set_operand_value(dst,d, i_type, thread, pI); -} - -void sqrt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case F32_TYPE: - if ( a.f32 < 0 ) - d.f32 = nanf(""); + assert(0); + break; + } + + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +void slct_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + ptx_reg_t a, b, c, d; + + unsigned i_type = pI->get_type(); + unsigned c_type = pI->get_type2(); + bool t = false; + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, c_type, thread, 1); + + switch (c_type) { + case S32_TYPE: + t = c.s32 >= 0; + break; + case F32_TYPE: + t = c.f32 >= 0; + break; + default: + assert(0); + } + + switch (i_type) { + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: + d.u16 = t ? a.u16 : b.u16; + break; + case F32_TYPE: + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: + d.u32 = t ? a.u32 : b.u32; + break; + case F64_TYPE: + case FF64_TYPE: + case B64_TYPE: + case S64_TYPE: + case U64_TYPE: + d.u64 = t ? a.u64 : b.u64; + break; + default: + assert(0); + } + + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +void sqrt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch (i_type) { + case F32_TYPE: + if (a.f32 < 0) + d.f32 = nanf(""); else - d.f32 = sqrt(a.f32); break; - case F64_TYPE: - case FF64_TYPE: - if ( a.f64 < 0 ) - d.f64 = nan(""); + d.f32 = sqrt(a.f32); + break; + case F64_TYPE: + case FF64_TYPE: + if (a.f64 < 0) + d.f64 = nan(""); else - d.f64 = sqrt(a.f64); break; - default: + d.f64 = sqrt(a.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } - - thread->set_operand_value(dst,d, i_type, thread, pI); -} - -void sst_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_instruction * cpI = const_cast(pI); // constant - const operand_info &dst = cpI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - unsigned type = pI->get_type(); - ptx_reg_t dst_data = thread->get_operand_value(dst, dst, type, thread, 1); - ptx_reg_t src1_data = thread->get_operand_value(src1, src1, type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, src1, type, thread, 1); - ptx_reg_t src3_data = thread->get_operand_value(src3, src1, type, thread, 1); - memory_space_t space = pI->get_space(); - memory_space *mem = NULL; - addr_t addr = src2_data.u32 * 4; // this assumes sstarr memory starts at address 0 - ptx_cta_info *cta_info = thread->m_cta_info; - - decode_space(space,thread,src1,mem,addr); - - size_t size; - int t; - type_info_key::type_decode(type,size,t); - - // store data in sstarr memory - mem->write(addr,size/8,&src3_data.s64,thread,pI); - - // sync threads - cpI->set_bar_id(16); // use 16 for sst because bar uses an int from 0-15 - - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; - thread->m_last_dram_callback.function = bar_callback; - thread->m_last_dram_callback.instruction = cpI; - - // the last thread that executes loads all of the data back from sstarr memory - int NUM_THREADS = cta_info->num_threads(); - cta_info->inc_bar_threads(); - if (NUM_THREADS == cta_info->get_bar_threads()) { - unsigned offset = 0; - addr = 0; - ptx_reg_t data; - float sstarr_fdata[NUM_THREADS]; - signed long long sstarr_ldata[NUM_THREADS]; - // loop through all of the threads - for (int tid = 0; tid < NUM_THREADS; tid++) { - data.u64=0; - mem->read(addr+(tid*4),size/8,&data.s64); - sstarr_fdata[tid] = data.f32; - sstarr_ldata[tid] = data.s64; - } - - // squeeze the zeros out of the array and store data back into original array - mem = NULL; - addr = src1_data.u32; - space.set_type(global_space); - decode_space(space,thread,src1,mem,addr); - // store nonzero entries and indices - for (int tid = 0; tid < NUM_THREADS; tid++) { - if (sstarr_fdata[tid] != 0) { - float ftid = (float)tid; - mem->write(addr+(offset*4),size/8,&sstarr_ldata[tid],thread,pI); - mem->write(addr+((NUM_THREADS+offset)*4),size/8,&ftid,thread,pI); - offset++; - } - } - // store the number of nonzero elements in the array - data = thread->get_operand_value(src1, dst, type, thread, 1); - data.s64 += 4*(offset-1); - thread->set_operand_value(dst, data, type, thread, pI); - - // fill the rest of the array with zeros (dst should always have a 0 in it) - while (offset < NUM_THREADS) { - mem->write(addr+(offset*4),size/8,&dst_data.s64,thread,pI); - offset++; - } - - cta_info->reset_bar_threads(); - thread->m_last_effective_address = addr+(NUM_THREADS-1)*4; - thread->m_last_memory_space = space; - } -} - -void ssy_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - //printf("Execution Warning: unimplemented ssy instruction is treated as a nop\n"); - // TODO: add implementation -} - -void st_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); //may be scalar or vector of regs - unsigned type = pI->get_type(); - ptx_reg_t addr_reg = thread->get_operand_value(dst, dst, type, thread, 1); - ptx_reg_t data; - memory_space_t space = pI->get_space(); - unsigned vector_spec = pI->get_vector(); - - memory_space *mem = NULL; - addr_t addr = addr_reg.u32; - - decode_space(space,thread,dst,mem,addr); - - size_t size; - int t; - type_info_key::type_decode(type,size,t); - - if (!vector_spec) { - data = thread->get_operand_value(src1, dst, type, thread, 1); - mem->write(addr,size/8,&data.s64,thread,pI); - } else { - if (vector_spec == V2_TYPE) { - ptx_reg_t* ptx_regs = new ptx_reg_t[2]; - thread->get_vector_operand_values(src1, ptx_regs, 2); - mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); - mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); - delete [] ptx_regs; - } - if (vector_spec == V3_TYPE) { - ptx_reg_t* ptx_regs = new ptx_reg_t[3]; - thread->get_vector_operand_values(src1, ptx_regs, 3); - mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); - mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); - mem->write(addr+2*size/8,size/8,&ptx_regs[2].s64,thread,pI); - delete [] ptx_regs; - } - if (vector_spec == V4_TYPE) { - ptx_reg_t* ptx_regs = new ptx_reg_t[4]; - thread->get_vector_operand_values(src1, ptx_regs, 4); - mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); - mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); - mem->write(addr+2*size/8,size/8,&ptx_regs[2].s64,thread,pI); - mem->write(addr+3*size/8,size/8,&ptx_regs[3].s64,thread,pI); - delete [] ptx_regs; + } + + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +void sst_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_instruction *cpI = const_cast(pI); // constant + const operand_info &dst = cpI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + unsigned type = pI->get_type(); + ptx_reg_t dst_data = thread->get_operand_value(dst, dst, type, thread, 1); + ptx_reg_t src1_data = thread->get_operand_value(src1, src1, type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, src1, type, thread, 1); + ptx_reg_t src3_data = thread->get_operand_value(src3, src1, type, thread, 1); + memory_space_t space = pI->get_space(); + memory_space *mem = NULL; + addr_t addr = + src2_data.u32 * 4; // this assumes sstarr memory starts at address 0 + ptx_cta_info *cta_info = thread->m_cta_info; + + decode_space(space, thread, src1, mem, addr); + + size_t size; + int t; + type_info_key::type_decode(type, size, t); + + // store data in sstarr memory + mem->write(addr, size / 8, &src3_data.s64, thread, pI); + + // sync threads + cpI->set_bar_id(16); // use 16 for sst because bar uses an int from 0-15 + + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; + thread->m_last_dram_callback.function = bar_callback; + thread->m_last_dram_callback.instruction = cpI; + + // the last thread that executes loads all of the data back from sstarr memory + int NUM_THREADS = cta_info->num_threads(); + cta_info->inc_bar_threads(); + if (NUM_THREADS == cta_info->get_bar_threads()) { + unsigned offset = 0; + addr = 0; + ptx_reg_t data; + float sstarr_fdata[NUM_THREADS]; + signed long long sstarr_ldata[NUM_THREADS]; + // loop through all of the threads + for (int tid = 0; tid < NUM_THREADS; tid++) { + data.u64 = 0; + mem->read(addr + (tid * 4), size / 8, &data.s64); + sstarr_fdata[tid] = data.f32; + sstarr_ldata[tid] = data.s64; + } + + // squeeze the zeros out of the array and store data back into original + // array + mem = NULL; + addr = src1_data.u32; + space.set_type(global_space); + decode_space(space, thread, src1, mem, addr); + // store nonzero entries and indices + for (int tid = 0; tid < NUM_THREADS; tid++) { + if (sstarr_fdata[tid] != 0) { + float ftid = (float)tid; + mem->write(addr + (offset * 4), size / 8, &sstarr_ldata[tid], thread, + pI); + mem->write(addr + ((NUM_THREADS + offset) * 4), size / 8, &ftid, thread, + pI); + offset++; } - } - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; + } + // store the number of nonzero elements in the array + data = thread->get_operand_value(src1, dst, type, thread, 1); + data.s64 += 4 * (offset - 1); + thread->set_operand_value(dst, data, type, thread, pI); + + // fill the rest of the array with zeros (dst should always have a 0 in it) + while (offset < NUM_THREADS) { + mem->write(addr + (offset * 4), size / 8, &dst_data.s64, thread, pI); + offset++; + } + + cta_info->reset_bar_threads(); + thread->m_last_effective_address = addr + (NUM_THREADS - 1) * 4; + thread->m_last_memory_space = space; + } +} + +void ssy_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // printf("Execution Warning: unimplemented ssy instruction is treated as a + // nop\n"); + // TODO: add implementation +} + +void st_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); // may be scalar or vector of regs + unsigned type = pI->get_type(); + ptx_reg_t addr_reg = thread->get_operand_value(dst, dst, type, thread, 1); + ptx_reg_t data; + memory_space_t space = pI->get_space(); + unsigned vector_spec = pI->get_vector(); + + memory_space *mem = NULL; + addr_t addr = addr_reg.u32; + + decode_space(space, thread, dst, mem, addr); + + size_t size; + int t; + type_info_key::type_decode(type, size, t); + + if (!vector_spec) { + data = thread->get_operand_value(src1, dst, type, thread, 1); + mem->write(addr, size / 8, &data.s64, thread, pI); + } else { + if (vector_spec == V2_TYPE) { + ptx_reg_t *ptx_regs = new ptx_reg_t[2]; + thread->get_vector_operand_values(src1, ptx_regs, 2); + mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); + mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); + delete[] ptx_regs; + } + if (vector_spec == V3_TYPE) { + ptx_reg_t *ptx_regs = new ptx_reg_t[3]; + thread->get_vector_operand_values(src1, ptx_regs, 3); + mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); + mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); + mem->write(addr + 2 * size / 8, size / 8, &ptx_regs[2].s64, thread, pI); + delete[] ptx_regs; + } + if (vector_spec == V4_TYPE) { + ptx_reg_t *ptx_regs = new ptx_reg_t[4]; + thread->get_vector_operand_values(src1, ptx_regs, 4); + mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); + mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); + mem->write(addr + 2 * size / 8, size / 8, &ptx_regs[2].s64, thread, pI); + mem->write(addr + 3 * size / 8, size / 8, &ptx_regs[3].s64, thread, pI); + delete[] ptx_regs; + } + } + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; } -void sub_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; - int overflow = 0; - int carry = 0; +void sub_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; + int overflow = 0; + int carry = 0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - //performs addition. Sets carry and overflow if needed. - //the constant is added in during subtraction so the carry bit is set properly. - switch ( i_type ) { - case S8_TYPE: + // performs addition. Sets carry and overflow if needed. + // the constant is added in during subtraction so the carry bit is set + // properly. + switch (i_type) { + case S8_TYPE: data.s64 = (src1_data.s64 & 0xFF) - (src2_data.s64 & 0xFF) + 0x100; - if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) != 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } - carry = (data.s32 & 0x100)>>8; + if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) != 0) { + overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; + } + carry = (data.s32 & 0x100) >> 8; break; - case S16_TYPE: + case S16_TYPE: data.s64 = (src1_data.s64 & 0xFFFF) - (src2_data.s64 & 0xFFFF) + 0x10000; - if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) != 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } - carry = (data.s32 & 0x10000)>>16; - break; - case S32_TYPE: - data.s64 = (src1_data.s64 & 0xFFFFFFFF) - (src2_data.s64 & 0xFFFFFFFF) + 0x100000000; - if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) != 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } - carry = ((data.u64)>>32) & 0x0001; - break; - case S64_TYPE: - data.s64 = src1_data.s64 - src2_data.s64; break; - case B8_TYPE: - case U8_TYPE: + if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) != 0) { + overflow = + ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; + } + carry = (data.s32 & 0x10000) >> 16; + break; + case S32_TYPE: + data.s64 = (src1_data.s64 & 0xFFFFFFFF) - (src2_data.s64 & 0xFFFFFFFF) + + 0x100000000; + if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) != 0) { + overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 + ? 0 + : 1; + } + carry = ((data.u64) >> 32) & 0x0001; + break; + case S64_TYPE: + data.s64 = src1_data.s64 - src2_data.s64; + break; + case B8_TYPE: + case U8_TYPE: data.u64 = (src1_data.u64 & 0xFF) - (src2_data.u64 & 0xFF) + 0x100; - carry = (data.u64 & 0x100)>>8; + carry = (data.u64 & 0x100) >> 8; break; - case B16_TYPE: - case U16_TYPE: + case B16_TYPE: + case U16_TYPE: data.u64 = (src1_data.u64 & 0xFFFF) - (src2_data.u64 & 0xFFFF) + 0x10000; - carry = (data.u64 & 0x10000)>>16; + carry = (data.u64 & 0x10000) >> 16; + break; + case B32_TYPE: + case U32_TYPE: + data.u64 = (src1_data.u64 & 0xFFFFFFFF) - (src2_data.u64 & 0xFFFFFFFF) + + 0x100000000; + carry = (data.u64 & 0x100000000) >> 32; break; - case B32_TYPE: - case U32_TYPE: - data.u64 = (src1_data.u64 & 0xFFFFFFFF) - (src2_data.u64 & 0xFFFFFFFF) + 0x100000000; - carry = (data.u64 & 0x100000000)>>32; + case B64_TYPE: + case U64_TYPE: + data.u64 = src1_data.u64 - src2_data.u64; break; - case B64_TYPE: - case U64_TYPE: - data.u64 = src1_data.u64 - src2_data.u64; break; - case F16_TYPE: data.f16 = src1_data.f16 - src2_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = src1_data.f32 - src2_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 - src2_data.f64; break; - default: assert(0); break; - } + case F16_TYPE: + data.f16 = src1_data.f16 - src2_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = src1_data.f32 - src2_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = src1_data.f64 - src2_data.f64; + break; + default: + assert(0); + break; + } - thread->set_operand_value(dst,data, i_type, thread, pI, overflow, carry); + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); } -void nop_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - // Do nothing +void nop_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // Do nothing } -void subc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void suld_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void sured_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void sust_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void suq_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } - +void subc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void suld_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void sured_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void sust_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void suq_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} union intfloat { - int a; - float b; + int a; + float b; }; -float reduce_precision( float x, unsigned bits ) -{ - intfloat tmp; - tmp.b = x; - int v = tmp.a; - int man = v & ((1<<23)-1); - int mask = ((1<= mx) nx -= elem_size; - unsigned ny = (y >= my)? my - 1 : y; - return nx + mx*ny; -} - -typedef unsigned (*texAddr_t) (unsigned x, unsigned y, unsigned mx, unsigned my, size_t elem_size); -float tex_linf_sampling(memory_space* mem, unsigned tex_array_base, - int x, int y, unsigned int width, unsigned int height, size_t elem_size, - float alpha, float beta, texAddr_t b_lim) -{ - float Tij; - float Ti1j; - float Tij1; - float Ti1j1; - - mem->read(tex_array_base + b_lim(x,y,width,height,elem_size), 4, &Tij); - mem->read(tex_array_base + b_lim(x+elem_size,y,width,height,elem_size), 4, &Ti1j); - mem->read(tex_array_base + b_lim(x,y+1,width,height,elem_size), 4, &Tij1); - mem->read(tex_array_base + b_lim(x+elem_size,y+1,width,height,elem_size), 4, &Ti1j1); - - float sample = (1-alpha)*(1-beta)*Tij + - alpha*(1-beta)*Ti1j + - (1-alpha)*beta*Tij1 + - alpha*beta*Ti1j1; - - return sample; -} - -float textureNormalizeElementSigned(int element, int bits) -{ - if (bits) { - int maxN = (1 << bits) - 1; - // removing upper bits - element &= maxN; - // normalizing the number to [-1.0,1.0] - maxN >>= 1; - float output = (float) element / maxN; - if (output < -1.0f) output = -1.0f; - return output; - } else { - return 0.0f; - } -} - -float textureNormalizeElementUnsigned(unsigned int element, int bits) -{ - if (bits) { - unsigned int maxN = (1 << bits) - 1; - // removing upper bits and normalizing the number to [0.0,1.0] - return (float)(element & maxN) / maxN; - } else { - return 0.0f; - } -} - -void textureNormalizeOutput( const struct cudaChannelFormatDesc& desc, ptx_reg_t& datax, ptx_reg_t& datay, ptx_reg_t& dataz, ptx_reg_t& dataw ) -{ - if (desc.f == cudaChannelFormatKindSigned) { - datax.f32 = textureNormalizeElementSigned( datax.s32, desc.x ); - datay.f32 = textureNormalizeElementSigned( datay.s32, desc.y ); - dataz.f32 = textureNormalizeElementSigned( dataz.s32, desc.z ); - dataw.f32 = textureNormalizeElementSigned( dataw.s32, desc.w ); - } else if (desc.f == cudaChannelFormatKindUnsigned) { - datax.f32 = textureNormalizeElementUnsigned( datax.u32, desc.x ); - datay.f32 = textureNormalizeElementUnsigned( datay.u32, desc.y ); - dataz.f32 = textureNormalizeElementUnsigned( dataz.u32, desc.z ); - dataw.f32 = textureNormalizeElementUnsigned( dataw.u32, desc.w ); - } else { - assert(0 && "Undefined texture read mode: cudaReadModeNormalizedFloat expect integer elements"); - } -} - -void tex_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - unsigned dimension = pI->dimension(); - const operand_info &dst = pI->dst(); //the registers to which fetched texel will be placed - const operand_info &src1 = pI->src1(); //the name of the texture - const operand_info &src2 = pI->src2(); //the vector registers containing coordinates of the texel to be fetched - - std::string texname = src1.name(); - unsigned to_type = pI->get_type(); - unsigned c_type = pI->get_type2(); - fflush(stdout); - ptx_reg_t data1, data2, data3, data4; - if (!thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs) - thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs = new ptx_reg_t[4]; - unsigned nelem = src2.get_vect_nelem(); - thread->get_vector_operand_values(src2, thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs, nelem); //ptx_reg should be 4 entry vector type...coordinates into texture - /* - For programs with many streams, textures can be bound and unbound - asynchronously. This means we need to use the kernel's "snapshot" of - the state of the texture mappings when it was launched (so that we - don't try to access the incorrect texture mapping if it's been updated, - or that we don't access a mapping that has been unbound). - */ - gpgpu_t *gpu = thread->get_gpu(); - kernel_info_t &k = thread->get_kernel(); - const struct textureReference* texref = gpu->get_texref(texname); - const struct cudaArray* cuArray = k.get_texarray(texname); - const struct textureInfo* texInfo = k.get_texinfo(texname); - const struct textureReferenceAttr* texAttr = gpu->get_texattr(texname); - - //assume always 2D f32 input - //access array with src2 coordinates - memory_space *mem = thread->get_global_memory(); - float x_f32, y_f32; - size_t size; - int t; - unsigned tex_array_base; - unsigned int width = 0, height = 0; - int x = 0; - int y = 0; - unsigned tex_array_index; - float alpha=0, beta=0; - - type_info_key::type_decode(to_type,size,t); - tex_array_base = cuArray->devPtr32; - - switch (dimension) { - case GEOM_MODIFIER_1D: +float reduce_precision(float x, unsigned bits) { + intfloat tmp; + tmp.b = x; + int v = tmp.a; + int man = v & ((1 << 23) - 1); + int mask = ((1 << bits) - 1) << (23 - bits); + int nv = (v & ((-1) - ((1 << 23) - 1))) | (mask & man); + tmp.a = nv; + float result = tmp.b; + return result; +} + +unsigned wrap(unsigned x, unsigned y, unsigned mx, unsigned my, + size_t elem_size) { + unsigned nx = (mx + x) % mx; + unsigned ny = (my + y) % my; + return nx + mx * ny; +} + +unsigned clamp(unsigned x, unsigned y, unsigned mx, unsigned my, + size_t elem_size) { + unsigned nx = x; + while (nx >= mx) nx -= elem_size; + unsigned ny = (y >= my) ? my - 1 : y; + return nx + mx * ny; +} + +typedef unsigned (*texAddr_t)(unsigned x, unsigned y, unsigned mx, unsigned my, + size_t elem_size); +float tex_linf_sampling(memory_space *mem, unsigned tex_array_base, int x, + int y, unsigned int width, unsigned int height, + size_t elem_size, float alpha, float beta, + texAddr_t b_lim) { + float Tij; + float Ti1j; + float Tij1; + float Ti1j1; + + mem->read(tex_array_base + b_lim(x, y, width, height, elem_size), 4, &Tij); + mem->read(tex_array_base + b_lim(x + elem_size, y, width, height, elem_size), + 4, &Ti1j); + mem->read(tex_array_base + b_lim(x, y + 1, width, height, elem_size), 4, + &Tij1); + mem->read( + tex_array_base + b_lim(x + elem_size, y + 1, width, height, elem_size), 4, + &Ti1j1); + + float sample = (1 - alpha) * (1 - beta) * Tij + alpha * (1 - beta) * Ti1j + + (1 - alpha) * beta * Tij1 + alpha * beta * Ti1j1; + + return sample; +} + +float textureNormalizeElementSigned(int element, int bits) { + if (bits) { + int maxN = (1 << bits) - 1; + // removing upper bits + element &= maxN; + // normalizing the number to [-1.0,1.0] + maxN >>= 1; + float output = (float)element / maxN; + if (output < -1.0f) output = -1.0f; + return output; + } else { + return 0.0f; + } +} + +float textureNormalizeElementUnsigned(unsigned int element, int bits) { + if (bits) { + unsigned int maxN = (1 << bits) - 1; + // removing upper bits and normalizing the number to [0.0,1.0] + return (float)(element & maxN) / maxN; + } else { + return 0.0f; + } +} + +void textureNormalizeOutput(const struct cudaChannelFormatDesc &desc, + ptx_reg_t &datax, ptx_reg_t &datay, + ptx_reg_t &dataz, ptx_reg_t &dataw) { + if (desc.f == cudaChannelFormatKindSigned) { + datax.f32 = textureNormalizeElementSigned(datax.s32, desc.x); + datay.f32 = textureNormalizeElementSigned(datay.s32, desc.y); + dataz.f32 = textureNormalizeElementSigned(dataz.s32, desc.z); + dataw.f32 = textureNormalizeElementSigned(dataw.s32, desc.w); + } else if (desc.f == cudaChannelFormatKindUnsigned) { + datax.f32 = textureNormalizeElementUnsigned(datax.u32, desc.x); + datay.f32 = textureNormalizeElementUnsigned(datay.u32, desc.y); + dataz.f32 = textureNormalizeElementUnsigned(dataz.u32, desc.z); + dataw.f32 = textureNormalizeElementUnsigned(dataw.u32, desc.w); + } else { + assert(0 && + "Undefined texture read mode: cudaReadModeNormalizedFloat expect " + "integer elements"); + } +} + +void tex_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + unsigned dimension = pI->dimension(); + const operand_info &dst = + pI->dst(); // the registers to which fetched texel will be placed + const operand_info &src1 = pI->src1(); // the name of the texture + const operand_info &src2 = pI->src2(); // the vector registers containing + // coordinates of the texel to be + // fetched + + std::string texname = src1.name(); + unsigned to_type = pI->get_type(); + unsigned c_type = pI->get_type2(); + fflush(stdout); + ptx_reg_t data1, data2, data3, data4; + if (!thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs) + thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs = new ptx_reg_t[4]; + unsigned nelem = src2.get_vect_nelem(); + thread->get_vector_operand_values( + src2, thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs, + nelem); // ptx_reg should be 4 entry vector type...coordinates into + // texture + /* + For programs with many streams, textures can be bound and unbound + asynchronously. This means we need to use the kernel's "snapshot" of + the state of the texture mappings when it was launched (so that we + don't try to access the incorrect texture mapping if it's been updated, + or that we don't access a mapping that has been unbound). + */ + gpgpu_t *gpu = thread->get_gpu(); + kernel_info_t &k = thread->get_kernel(); + const struct textureReference *texref = gpu->get_texref(texname); + const struct cudaArray *cuArray = k.get_texarray(texname); + const struct textureInfo *texInfo = k.get_texinfo(texname); + const struct textureReferenceAttr *texAttr = gpu->get_texattr(texname); + + // assume always 2D f32 input + // access array with src2 coordinates + memory_space *mem = thread->get_global_memory(); + float x_f32, y_f32; + size_t size; + int t; + unsigned tex_array_base; + unsigned int width = 0, height = 0; + int x = 0; + int y = 0; + unsigned tex_array_index; + float alpha = 0, beta = 0; + + type_info_key::type_decode(to_type, size, t); + tex_array_base = cuArray->devPtr32; + + switch (dimension) { + case GEOM_MODIFIER_1D: width = cuArray->width; height = cuArray->height; if (texref->normalized) { - assert(c_type == F32_TYPE); - x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - if (texref->addressMode[0] == cudaAddressModeClamp) { - x_f32 = (x_f32 > 1.0)? 1.0 : x_f32; - x_f32 = (x_f32 < 0.0)? 0.0 : x_f32; - } else if (texref->addressMode[0] == cudaAddressModeWrap) { - x_f32 = x_f32 - floor(x_f32); - } - - if( texref->filterMode == cudaFilterModeLinear ) { - float xb = x_f32 * width - 0.5; - alpha = xb - floor(xb); - alpha = reduce_precision(alpha,9); - beta = 0.0; - - x = (int)floor(xb); - y = 0; - } else { - x = (int) floor(x_f32 * width); - y = 0; - } + assert(c_type == F32_TYPE); + x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; + if (texref->addressMode[0] == cudaAddressModeClamp) { + x_f32 = (x_f32 > 1.0) ? 1.0 : x_f32; + x_f32 = (x_f32 < 0.0) ? 0.0 : x_f32; + } else if (texref->addressMode[0] == cudaAddressModeWrap) { + x_f32 = x_f32 - floor(x_f32); + } + + if (texref->filterMode == cudaFilterModeLinear) { + float xb = x_f32 * width - 0.5; + alpha = xb - floor(xb); + alpha = reduce_precision(alpha, 9); + beta = 0.0; + + x = (int)floor(xb); + y = 0; + } else { + x = (int)floor(x_f32 * width); + y = 0; + } } else { - switch ( c_type ) { - case S32_TYPE: + switch (c_type) { + case S32_TYPE: x = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].s32; - assert(texref->filterMode == cudaFilterModePoint); - break; - case F32_TYPE: + assert(texref->filterMode == cudaFilterModePoint); + break; + case F32_TYPE: x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - alpha = x_f32 - floor(x_f32); // offset into subtexel (for linear sampling) - x = (int) x_f32; - break; - default: assert(0 && "Unsupported texture coordinate type."); - } - // handle texture fetch that exceeded boundaries - if (texref->addressMode[0] == cudaAddressModeClamp) { - x = (x > width - 1)? (width - 1) : x; - x = (x < 0)? 0 : x; - } else if (texref->addressMode[0] == cudaAddressModeWrap) { - x = x % width; - } + alpha = x_f32 - + floor(x_f32); // offset into subtexel (for linear sampling) + x = (int)x_f32; + break; + default: + assert(0 && "Unsupported texture coordinate type."); + } + // handle texture fetch that exceeded boundaries + if (texref->addressMode[0] == cudaAddressModeClamp) { + x = (x > width - 1) ? (width - 1) : x; + x = (x < 0) ? 0 : x; + } else if (texref->addressMode[0] == cudaAddressModeWrap) { + x = x % width; + } } - width *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; - x *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; + width *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z) / + 8; + x *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z) / + 8; tex_array_index = tex_array_base + x; break; - case GEOM_MODIFIER_2D: + case GEOM_MODIFIER_2D: width = cuArray->width; height = cuArray->height; if (texref->normalized) { - x_f32 = reduce_precision(thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32,16); - y_f32 = reduce_precision(thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32,15); - - if (texref->addressMode[0]) {//clamp - if (x_f32<0) x_f32 = 0; - if (x_f32>=1) x_f32 = 1 - 1/x_f32; - } else {//wrap - x_f32 = x_f32 - floor(x_f32); - } - if (texref->addressMode[1]) {//clamp - if (y_f32<0) y_f32 = 0; - if (y_f32>=1) y_f32 = 1 - 1/y_f32; - } else {//wrap - y_f32 = y_f32 - floor(y_f32); - } - - if( texref->filterMode == cudaFilterModeLinear ) { - float xb = x_f32 * width - 0.5; - float yb = y_f32 * height - 0.5; - alpha = xb - floor(xb); - beta = yb - floor(yb); - alpha = reduce_precision(alpha,9); - beta = reduce_precision(beta,9); - - x = (int)floor(xb); - y = (int)floor(yb); - } else { - x = (int) floor(x_f32 * width); - y = (int) floor(y_f32 * height); - } + x_f32 = reduce_precision( + thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32, 16); + y_f32 = reduce_precision( + thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32, 15); + + if (texref->addressMode[0]) { // clamp + if (x_f32 < 0) x_f32 = 0; + if (x_f32 >= 1) x_f32 = 1 - 1 / x_f32; + } else { // wrap + x_f32 = x_f32 - floor(x_f32); + } + if (texref->addressMode[1]) { // clamp + if (y_f32 < 0) y_f32 = 0; + if (y_f32 >= 1) y_f32 = 1 - 1 / y_f32; + } else { // wrap + y_f32 = y_f32 - floor(y_f32); + } + + if (texref->filterMode == cudaFilterModeLinear) { + float xb = x_f32 * width - 0.5; + float yb = y_f32 * height - 0.5; + alpha = xb - floor(xb); + beta = yb - floor(yb); + alpha = reduce_precision(alpha, 9); + beta = reduce_precision(beta, 9); + + x = (int)floor(xb); + y = (int)floor(yb); + } else { + x = (int)floor(x_f32 * width); + y = (int)floor(y_f32 * height); + } } else { - x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - y_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32; - - alpha = x_f32 - floor(x_f32); - beta = y_f32 - floor(y_f32); - - x = (int) x_f32; - y = (int) y_f32; - if (texref->addressMode[0]) {//clamp - if (x<0) x = 0; - if (x>= (int)width) x = width-1; - } else {//wrap - x = x % width; - if (x < 0) x*= -1; - } - if (texref->addressMode[1]) {//clamp - if (y<0) y = 0; - if (y>= (int)height) y = height -1; - } else {//wrap - y = y % height; - if (y < 0) y *= -1; - } + x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; + y_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32; + + alpha = x_f32 - floor(x_f32); + beta = y_f32 - floor(y_f32); + + x = (int)x_f32; + y = (int)y_f32; + if (texref->addressMode[0]) { // clamp + if (x < 0) x = 0; + if (x >= (int)width) x = width - 1; + } else { // wrap + x = x % width; + if (x < 0) x *= -1; + } + if (texref->addressMode[1]) { // clamp + if (y < 0) y = 0; + if (y >= (int)height) y = height - 1; + } else { // wrap + y = y % height; + if (y < 0) y *= -1; + } } - width *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; - x *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; - tex_array_index = tex_array_base + (x + width*y); - break; - default: - assert(0); break; - } - switch ( to_type ) { - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case B8_TYPE: - case B16_TYPE: - case B32_TYPE: - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: { - unsigned long long elementOffset = 0; // offset into the next element - mem->read( tex_array_index, cuArray->desc.x/8, &data1.u32); - elementOffset += cuArray->desc.x/8; + width *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z) / + 8; + x *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z) / + 8; + tex_array_index = tex_array_base + (x + width * y); + break; + default: + assert(0); + break; + } + switch (to_type) { + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case B8_TYPE: + case B16_TYPE: + case B32_TYPE: + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: { + unsigned long long elementOffset = 0; // offset into the next element + mem->read(tex_array_index, cuArray->desc.x / 8, &data1.u32); + elementOffset += cuArray->desc.x / 8; if (cuArray->desc.y) { - mem->read( tex_array_index + elementOffset, cuArray->desc.y/8, &data2.u32); - elementOffset += cuArray->desc.y/8; - if (cuArray->desc.z) { - mem->read( tex_array_index + elementOffset, cuArray->desc.z/8, &data3.u32); - elementOffset += cuArray->desc.z/8; - if (cuArray->desc.w) - mem->read( tex_array_index + elementOffset, cuArray->desc.w/8, &data4.u32); - } + mem->read(tex_array_index + elementOffset, cuArray->desc.y / 8, + &data2.u32); + elementOffset += cuArray->desc.y / 8; + if (cuArray->desc.z) { + mem->read(tex_array_index + elementOffset, cuArray->desc.z / 8, + &data3.u32); + elementOffset += cuArray->desc.z / 8; + if (cuArray->desc.w) + mem->read(tex_array_index + elementOffset, cuArray->desc.w / 8, + &data4.u32); + } } break; - } - case B64_TYPE: - case U64_TYPE: - case S64_TYPE: - mem->read( tex_array_index, 8, &data1.u64); + } + case B64_TYPE: + case U64_TYPE: + case S64_TYPE: + mem->read(tex_array_index, 8, &data1.u64); if (cuArray->desc.y) { - mem->read( tex_array_index+8, 8, &data2.u64); - if (cuArray->desc.z) { - mem->read( tex_array_index+16, 8, &data3.u64); - if (cuArray->desc.w) - mem->read( tex_array_index+24, 8, &data4.u64); - } + mem->read(tex_array_index + 8, 8, &data2.u64); + if (cuArray->desc.z) { + mem->read(tex_array_index + 16, 8, &data3.u64); + if (cuArray->desc.w) mem->read(tex_array_index + 24, 8, &data4.u64); + } } break; - case F16_TYPE: assert(0); break; - case F32_TYPE: { - if( texref->filterMode == cudaFilterModeLinear ) { - texAddr_t b_lim = wrap; - if ( texref->addressMode[0] == cudaAddressModeClamp ) { - b_lim = clamp; - } - size_t elem_size = (cuArray->desc.x + cuArray->desc.y + cuArray->desc.z + cuArray->desc.w) / 8; - size_t elem_ofst = 0; - - data1.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.x / 8; - if (cuArray->desc.y) { - data2.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.y / 8; - if (cuArray->desc.z) { - data3.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.z / 8; - if (cuArray->desc.w) - data4.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); - } - } + case F16_TYPE: + assert(0); + break; + case F32_TYPE: { + if (texref->filterMode == cudaFilterModeLinear) { + texAddr_t b_lim = wrap; + if (texref->addressMode[0] == cudaAddressModeClamp) { + b_lim = clamp; + } + size_t elem_size = (cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z + cuArray->desc.w) / + 8; + size_t elem_ofst = 0; + + data1.f32 = + tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, + height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.x / 8; + if (cuArray->desc.y) { + data2.f32 = + tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, + height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.y / 8; + if (cuArray->desc.z) { + data3.f32 = + tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, + height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.z / 8; + if (cuArray->desc.w) + data4.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, + y, width, height, elem_size, alpha, + beta, b_lim); + } + } } else { - mem->read( tex_array_index, cuArray->desc.x/8, &data1.f32); - if (cuArray->desc.y) { - mem->read( tex_array_index+4, cuArray->desc.y/8, &data2.f32); - if (cuArray->desc.z) { - mem->read( tex_array_index+8, cuArray->desc.z/8, &data3.f32); - if (cuArray->desc.w) - mem->read( tex_array_index+12, cuArray->desc.w/8, &data4.f32); - } - } + mem->read(tex_array_index, cuArray->desc.x / 8, &data1.f32); + if (cuArray->desc.y) { + mem->read(tex_array_index + 4, cuArray->desc.y / 8, &data2.f32); + if (cuArray->desc.z) { + mem->read(tex_array_index + 8, cuArray->desc.z / 8, &data3.f32); + if (cuArray->desc.w) + mem->read(tex_array_index + 12, cuArray->desc.w / 8, &data4.f32); + } + } } - } break; - case F64_TYPE: - case FF64_TYPE: - mem->read( tex_array_index, 8, &data1.f64); + } break; + case F64_TYPE: + case FF64_TYPE: + mem->read(tex_array_index, 8, &data1.f64); if (cuArray->desc.y) { - mem->read( tex_array_index+8, 8, &data2.f64); - if (cuArray->desc.z) { - mem->read( tex_array_index+16, 8, &data3.f64); - if (cuArray->desc.w) - mem->read( tex_array_index+24, 8, &data4.f64); - } + mem->read(tex_array_index + 8, 8, &data2.f64); + if (cuArray->desc.z) { + mem->read(tex_array_index + 16, 8, &data3.f64); + if (cuArray->desc.w) mem->read(tex_array_index + 24, 8, &data4.f64); + } } break; - default: assert(0); break; - } - int x_block_coord, y_block_coord, memreqindex, blockoffset; + default: + assert(0); + break; + } + int x_block_coord, y_block_coord, memreqindex, blockoffset; - switch (dimension) { - case GEOM_MODIFIER_1D: + switch (dimension) { + case GEOM_MODIFIER_1D: thread->m_last_effective_address = tex_array_index; break; - case GEOM_MODIFIER_2D: + case GEOM_MODIFIER_2D: x_block_coord = x >> (texInfo->Tx_numbits + texInfo->texel_size_numbits); y_block_coord = y >> texInfo->Ty_numbits; - memreqindex = ((y_block_coord*cuArray->width/texInfo->Tx)+x_block_coord)<<6; + memreqindex = + ((y_block_coord * cuArray->width / texInfo->Tx) + x_block_coord) << 6; - blockoffset = (x%(texInfo->Tx*texInfo->texel_size) + (y%(texInfo->Ty)<<(texInfo->Tx_numbits + texInfo->texel_size_numbits))); + blockoffset = (x % (texInfo->Tx * texInfo->texel_size) + + (y % (texInfo->Ty) + << (texInfo->Tx_numbits + texInfo->texel_size_numbits))); memreqindex += blockoffset; - thread->m_last_effective_address = tex_array_base + memreqindex;//tex_array_index; + thread->m_last_effective_address = + tex_array_base + memreqindex; // tex_array_index; break; - default: + default: assert(0); - } - thread->m_last_memory_space = tex_space; - - // normalize output into floating point numbers according to the texture read mode - if (texAttr->m_readmode == cudaReadModeNormalizedFloat) { - textureNormalizeOutput(cuArray->desc, data1, data2, data3, data4); - } else { - assert(texAttr->m_readmode == cudaReadModeElementType); - } - - thread->set_vector_operand_values(dst,data1,data2,data3,data4); -} - -void txq_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void trap_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vabsdiff_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vadd_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vmad_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vmax_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vmin_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vset_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vshl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vshr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vsub_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } - -void vote_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - static bool first_in_warp = true; - static bool and_all; - static bool or_all; - static unsigned int ballot_result; - static std::list threads_in_warp; - static unsigned last_tid; - - if( first_in_warp ) { - first_in_warp = false; - threads_in_warp.clear(); - and_all = true; - or_all = false; - ballot_result = 0; - int offset=31; - while( (offset>=0) && !pI->active(offset) ) - offset--; - assert( offset >= 0 ); - last_tid = (thread->get_hw_tid() - (thread->get_hw_tid()%pI->warp_size())) + offset; - } - - ptx_reg_t src1_data; - const operand_info &src1 = pI->src1(); - src1_data = thread->get_operand_value(src1, pI->dst(), PRED_TYPE, thread, 1); - - //predicate value was changed so the lowest bit being set means the zero flag is set. - //As a result, the value of src1_data.pred must be inverted to get proper behavior - bool pred_value = !(src1_data.pred & 0x0001); - bool invert = src1.is_neg_pred(); - - threads_in_warp.push_back(thread); - and_all &= (invert ^ pred_value); - or_all |= (invert ^ pred_value); - - // vote.ballot - if (invert ^ pred_value) { - int lane_id = thread->get_hw_tid() % pI->warp_size(); - ballot_result |= (1 << lane_id); - } - - if( thread->get_hw_tid() == last_tid ) { - if (pI->vote_mode() == ptx_instruction::vote_ballot) { - ptx_reg_t data = ballot_result; - for( std::list::iterator t=threads_in_warp.begin(); t!=threads_in_warp.end(); ++t ) { - const operand_info &dst = pI->dst(); - (*t)->set_operand_value(dst,data, pI->get_type(), (*t), pI); - } - } else { - bool pred_value = false; + } + thread->m_last_memory_space = tex_space; + + // normalize output into floating point numbers according to the texture read + // mode + if (texAttr->m_readmode == cudaReadModeNormalizedFloat) { + textureNormalizeOutput(cuArray->desc, data1, data2, data3, data4); + } else { + assert(texAttr->m_readmode == cudaReadModeElementType); + } - switch( pI->vote_mode() ) { - case ptx_instruction::vote_any: pred_value = or_all; break; - case ptx_instruction::vote_all: pred_value = and_all; break; - case ptx_instruction::vote_uni: pred_value = (or_all ^ and_all); break; - default: - abort(); - } - ptx_reg_t data; - data.pred = pred_value?0:1; //the way ptxplus handles the zero flag, 1 = false and 0 = true - - for( std::list::iterator t=threads_in_warp.begin(); t!=threads_in_warp.end(); ++t ) { - const operand_info &dst = pI->dst(); - (*t)->set_operand_value(dst,data, PRED_TYPE, (*t), pI); - } + thread->set_vector_operand_values(dst, data1, data2, data3, data4); +} + +void txq_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void trap_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vabsdiff_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vadd_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vmad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vmax_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vmin_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vset_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vshl_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vshr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vsub_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} + +void vote_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + static bool first_in_warp = true; + static bool and_all; + static bool or_all; + static unsigned int ballot_result; + static std::list threads_in_warp; + static unsigned last_tid; + + if (first_in_warp) { + first_in_warp = false; + threads_in_warp.clear(); + and_all = true; + or_all = false; + ballot_result = 0; + int offset = 31; + while ((offset >= 0) && !pI->active(offset)) offset--; + assert(offset >= 0); + last_tid = + (thread->get_hw_tid() - (thread->get_hw_tid() % pI->warp_size())) + + offset; + } + + ptx_reg_t src1_data; + const operand_info &src1 = pI->src1(); + src1_data = thread->get_operand_value(src1, pI->dst(), PRED_TYPE, thread, 1); + + // predicate value was changed so the lowest bit being set means the zero flag + // is set. + // As a result, the value of src1_data.pred must be inverted to get proper + // behavior + bool pred_value = !(src1_data.pred & 0x0001); + bool invert = src1.is_neg_pred(); + + threads_in_warp.push_back(thread); + and_all &= (invert ^ pred_value); + or_all |= (invert ^ pred_value); + + // vote.ballot + if (invert ^ pred_value) { + int lane_id = thread->get_hw_tid() % pI->warp_size(); + ballot_result |= (1 << lane_id); + } + + if (thread->get_hw_tid() == last_tid) { + if (pI->vote_mode() == ptx_instruction::vote_ballot) { + ptx_reg_t data = ballot_result; + for (std::list::iterator t = threads_in_warp.begin(); + t != threads_in_warp.end(); ++t) { + const operand_info &dst = pI->dst(); + (*t)->set_operand_value(dst, data, pI->get_type(), (*t), pI); } - first_in_warp = true; - } -} - -void xor_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) ^ ~(src2_data.pred)); - else - data.u64 = src1_data.u64 ^ src2_data.u64; - - thread->set_operand_value(dst,data, i_type, thread, pI); -} - -void inst_not_implemented( const ptx_instruction * pI ) -{ - printf("GPGPU-Sim PTX: ERROR (%s:%u) instruction \"%s\" not (yet) implemented\n", - pI->source_file(), - pI->source_line(), - pI->get_opcode_cstr() ); - abort(); -} - -ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread) -{ - ptx_reg_t result; - memory_space *mem = NULL; - size_t size; - int t; - result.u64=0; - - //complete other cases for reading from memory, such as reading from other const memory - if(opInfo.get_addr_space() == global_space) - { - mem = thread->get_global_memory(); - type_info_key::type_decode(type,size,t); - mem->read(opData.u32,size/8,&result.u64); - if( type == S16_TYPE || type == S32_TYPE ) - sign_extend(result,size,dstInfo); - } - else if(opInfo.get_addr_space() == shared_space) - { - mem = thread->m_shared_mem; - type_info_key::type_decode(type,size,t); - mem->read(opData.u32,size/8,&result.u64); - - if( type == S16_TYPE || type == S32_TYPE ) - sign_extend(result,size,dstInfo); - - } - else if(opInfo.get_addr_space() == const_space) - { - mem = thread->get_global_memory(); - type_info_key::type_decode(type,size,t); - - mem->read((opData.u32 + opInfo.get_const_mem_offset()),size/8,&result.u64); - - if( type == S16_TYPE || type == S32_TYPE ) - sign_extend(result,size,dstInfo); - } - else - { - result = opData; - } - - if(opInfo.get_operand_lohi() == 1) - { - result.u64 = result.u64 & 0xFFFF; - } - else if(opInfo.get_operand_lohi() == 2) - { - result.u64 = (result.u64>>16) & 0xFFFF; - } + } else { + bool pred_value = false; + + switch (pI->vote_mode()) { + case ptx_instruction::vote_any: + pred_value = or_all; + break; + case ptx_instruction::vote_all: + pred_value = and_all; + break; + case ptx_instruction::vote_uni: + pred_value = (or_all ^ and_all); + break; + default: + abort(); + } + ptx_reg_t data; + data.pred = pred_value ? 0 : 1; // the way ptxplus handles the zero flag, + // 1 = false and 0 = true + + for (std::list::iterator t = threads_in_warp.begin(); + t != threads_in_warp.end(); ++t) { + const operand_info &dst = pI->dst(); + (*t)->set_operand_value(dst, data, PRED_TYPE, (*t), pI); + } + } + first_in_warp = true; + } +} + +void xor_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) ^ ~(src2_data.pred)); + else + data.u64 = src1_data.u64 ^ src2_data.u64; - if(opInfo.get_operand_neg() == true) { - result.f32 = -result.f32; - } + thread->set_operand_value(dst, data, i_type, thread, pI); +} - return result; +void inst_not_implemented(const ptx_instruction *pI) { + printf( + "GPGPU-Sim PTX: ERROR (%s:%u) instruction \"%s\" not (yet) implemented\n", + pI->source_file(), pI->source_line(), pI->get_opcode_cstr()); + abort(); } +ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, + operand_info dstInfo, unsigned type, + ptx_thread_info *thread) { + ptx_reg_t result; + memory_space *mem = NULL; + size_t size; + int t; + result.u64 = 0; + + // complete other cases for reading from memory, such as reading from other + // const memory + if (opInfo.get_addr_space() == global_space) { + mem = thread->get_global_memory(); + type_info_key::type_decode(type, size, t); + mem->read(opData.u32, size / 8, &result.u64); + if (type == S16_TYPE || type == S32_TYPE) + sign_extend(result, size, dstInfo); + } else if (opInfo.get_addr_space() == shared_space) { + mem = thread->m_shared_mem; + type_info_key::type_decode(type, size, t); + mem->read(opData.u32, size / 8, &result.u64); + + if (type == S16_TYPE || type == S32_TYPE) + sign_extend(result, size, dstInfo); + + } else if (opInfo.get_addr_space() == const_space) { + mem = thread->get_global_memory(); + type_info_key::type_decode(type, size, t); + + mem->read((opData.u32 + opInfo.get_const_mem_offset()), size / 8, + &result.u64); + + if (type == S16_TYPE || type == S32_TYPE) + sign_extend(result, size, dstInfo); + } else { + result = opData; + } + + if (opInfo.get_operand_lohi() == 1) { + result.u64 = result.u64 & 0xFFFF; + } else if (opInfo.get_operand_lohi() == 2) { + result.u64 = (result.u64 >> 16) & 0xFFFF; + } + + if (opInfo.get_operand_neg() == true) { + result.f32 = -result.f32; + } + + return result; +} diff --git a/src/cuda-sim/memory.cc b/src/cuda-sim/memory.cc index 4b2acdf..b630da9 100644 --- a/src/cuda-sim/memory.cc +++ b/src/cuda-sim/memory.cc @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -27,190 +29,206 @@ #include "memory.h" #include -#include "../debug.h" #include "../../libcuda/gpgpu_context.h" +#include "../debug.h" -template memory_space_impl::memory_space_impl( std::string name, unsigned hash_size ) -{ - m_name = name; - MEM_MAP_RESIZE(hash_size); - - m_log2_block_size = -1; - for( unsigned n=0, mask=1; mask != 0; mask <<= 1, n++ ) { - if( BSIZE & mask ) { - assert( m_log2_block_size == (unsigned)-1 ); - m_log2_block_size = n; - } - } - assert( m_log2_block_size != (unsigned)-1 ); +template +memory_space_impl::memory_space_impl(std::string name, + unsigned hash_size) { + m_name = name; + MEM_MAP_RESIZE(hash_size); + + m_log2_block_size = -1; + for (unsigned n = 0, mask = 1; mask != 0; mask <<= 1, n++) { + if (BSIZE & mask) { + assert(m_log2_block_size == (unsigned)-1); + m_log2_block_size = n; + } + } + assert(m_log2_block_size != (unsigned)-1); } -template void memory_space_impl::write_only( mem_addr_t offset, mem_addr_t index, size_t length, const void *data) -{ - m_data[index].write(offset,length,(const unsigned char*)data); +template +void memory_space_impl::write_only(mem_addr_t offset, mem_addr_t index, + size_t length, const void *data) { + m_data[index].write(offset, length, (const unsigned char *)data); } -template void memory_space_impl::write( mem_addr_t addr, size_t length, const void *data, class ptx_thread_info *thd, const ptx_instruction *pI) -{ - - mem_addr_t index = addr >> m_log2_block_size; - - if ( (addr+length) <= (index+1)*BSIZE ) { - // fast route for intra-block access - unsigned offset = addr & (BSIZE-1); - unsigned nbytes = length; - m_data[index].write(offset,nbytes,(const unsigned char*)data); - } else { - // slow route for inter-block access - unsigned nbytes_remain = length; - unsigned src_offset = 0; - mem_addr_t current_addr = addr; - - while (nbytes_remain > 0) { - unsigned offset = current_addr & (BSIZE-1); - mem_addr_t page = current_addr >> m_log2_block_size; - mem_addr_t access_limit = offset + nbytes_remain; - if (access_limit > BSIZE) { - access_limit = BSIZE; - } - - size_t tx_bytes = access_limit - offset; - m_data[page].write(offset, tx_bytes, &((const unsigned char*)data)[src_offset]); - - // advance pointers - src_offset += tx_bytes; - current_addr += tx_bytes; - nbytes_remain -= tx_bytes; - } - assert(nbytes_remain == 0); - } - if( !m_watchpoints.empty() ) { - std::map::iterator i; - for( i=m_watchpoints.begin(); i!=m_watchpoints.end(); i++ ) { - mem_addr_t wa = i->second; - if( ((addr<=wa) && ((addr+length)>wa)) || ((addr>wa) && (addr < (wa+4))) ) - thd->get_gpu()->gpgpu_ctx->the_gpgpusim->g_the_gpu->hit_watchpoint(i->first,thd,pI); +template +void memory_space_impl::write(mem_addr_t addr, size_t length, + const void *data, + class ptx_thread_info *thd, + const ptx_instruction *pI) { + mem_addr_t index = addr >> m_log2_block_size; + + if ((addr + length) <= (index + 1) * BSIZE) { + // fast route for intra-block access + unsigned offset = addr & (BSIZE - 1); + unsigned nbytes = length; + m_data[index].write(offset, nbytes, (const unsigned char *)data); + } else { + // slow route for inter-block access + unsigned nbytes_remain = length; + unsigned src_offset = 0; + mem_addr_t current_addr = addr; + + while (nbytes_remain > 0) { + unsigned offset = current_addr & (BSIZE - 1); + mem_addr_t page = current_addr >> m_log2_block_size; + mem_addr_t access_limit = offset + nbytes_remain; + if (access_limit > BSIZE) { + access_limit = BSIZE; } - } + + size_t tx_bytes = access_limit - offset; + m_data[page].write(offset, tx_bytes, + &((const unsigned char *)data)[src_offset]); + + // advance pointers + src_offset += tx_bytes; + current_addr += tx_bytes; + nbytes_remain -= tx_bytes; + } + assert(nbytes_remain == 0); + } + if (!m_watchpoints.empty()) { + std::map::iterator i; + for (i = m_watchpoints.begin(); i != m_watchpoints.end(); i++) { + mem_addr_t wa = i->second; + if (((addr <= wa) && ((addr + length) > wa)) || + ((addr > wa) && (addr < (wa + 4)))) + thd->get_gpu()->gpgpu_ctx->the_gpgpusim->g_the_gpu->hit_watchpoint( + i->first, thd, pI); + } + } } -template void memory_space_impl::read_single_block( mem_addr_t blk_idx, mem_addr_t addr, size_t length, void *data) const -{ - if ((addr + length) > (blk_idx + 1) * BSIZE) { - printf("GPGPU-Sim PTX: ERROR * access to memory \'%s\' is unaligned : addr=0x%x, length=%zu\n", - m_name.c_str(), addr, length); - printf("GPGPU-Sim PTX: (addr+length)=0x%lx > 0x%x=(index+1)*BSIZE, index=0x%x, BSIZE=0x%x\n", - (addr+length),(blk_idx+1)*BSIZE, blk_idx, BSIZE); - throw 1; - } - typename map_t::const_iterator i = m_data.find(blk_idx); - if( i == m_data.end() ) { - for( size_t n=0; n < length; n++ ) - ((unsigned char*)data)[n] = (unsigned char) 0; - //printf("GPGPU-Sim PTX: WARNING reading %zu bytes from unititialized memory at address 0x%x in space %s\n", length, addr, m_name.c_str() ); - } else { - unsigned offset = addr & (BSIZE-1); - unsigned nbytes = length; - i->second.read(offset,nbytes,(unsigned char*)data); - } +template +void memory_space_impl::read_single_block(mem_addr_t blk_idx, + mem_addr_t addr, size_t length, + void *data) const { + if ((addr + length) > (blk_idx + 1) * BSIZE) { + printf( + "GPGPU-Sim PTX: ERROR * access to memory \'%s\' is unaligned : " + "addr=0x%x, length=%zu\n", + m_name.c_str(), addr, length); + printf( + "GPGPU-Sim PTX: (addr+length)=0x%lx > 0x%x=(index+1)*BSIZE, " + "index=0x%x, BSIZE=0x%x\n", + (addr + length), (blk_idx + 1) * BSIZE, blk_idx, BSIZE); + throw 1; + } + typename map_t::const_iterator i = m_data.find(blk_idx); + if (i == m_data.end()) { + for (size_t n = 0; n < length; n++) + ((unsigned char *)data)[n] = (unsigned char)0; + // printf("GPGPU-Sim PTX: WARNING reading %zu bytes from unititialized + // memory at address 0x%x in space %s\n", length, addr, m_name.c_str() ); + } else { + unsigned offset = addr & (BSIZE - 1); + unsigned nbytes = length; + i->second.read(offset, nbytes, (unsigned char *)data); + } } -template void memory_space_impl::read( mem_addr_t addr, size_t length, void *data ) const -{ - mem_addr_t index = addr >> m_log2_block_size; - if ((addr+length) <= (index+1)*BSIZE ) { - // fast route for intra-block access - read_single_block(index, addr, length, data); - } else { - // slow route for inter-block access - unsigned nbytes_remain = length; - unsigned dst_offset = 0; - mem_addr_t current_addr = addr; - - while (nbytes_remain > 0) { - unsigned offset = current_addr & (BSIZE-1); - mem_addr_t page = current_addr >> m_log2_block_size; - mem_addr_t access_limit = offset + nbytes_remain; - if (access_limit > BSIZE) { - access_limit = BSIZE; - } - - size_t tx_bytes = access_limit - offset; - read_single_block(page, current_addr, tx_bytes, &((unsigned char*)data)[dst_offset]); - - // advance pointers - dst_offset += tx_bytes; - current_addr += tx_bytes; - nbytes_remain -= tx_bytes; +template +void memory_space_impl::read(mem_addr_t addr, size_t length, + void *data) const { + mem_addr_t index = addr >> m_log2_block_size; + if ((addr + length) <= (index + 1) * BSIZE) { + // fast route for intra-block access + read_single_block(index, addr, length, data); + } else { + // slow route for inter-block access + unsigned nbytes_remain = length; + unsigned dst_offset = 0; + mem_addr_t current_addr = addr; + + while (nbytes_remain > 0) { + unsigned offset = current_addr & (BSIZE - 1); + mem_addr_t page = current_addr >> m_log2_block_size; + mem_addr_t access_limit = offset + nbytes_remain; + if (access_limit > BSIZE) { + access_limit = BSIZE; } - assert(nbytes_remain == 0); - } + + size_t tx_bytes = access_limit - offset; + read_single_block(page, current_addr, tx_bytes, + &((unsigned char *)data)[dst_offset]); + + // advance pointers + dst_offset += tx_bytes; + current_addr += tx_bytes; + nbytes_remain -= tx_bytes; + } + assert(nbytes_remain == 0); + } } -template void memory_space_impl::print( const char *format, FILE *fout ) const -{ - typename map_t::const_iterator i_page; +template +void memory_space_impl::print(const char *format, FILE *fout) const { + typename map_t::const_iterator i_page; - for ( i_page = m_data.begin(); i_page != m_data.end(); ++i_page) { - fprintf(fout, "%s %08x:", m_name.c_str(), i_page->first); - i_page->second.print(format, fout); - } + for (i_page = m_data.begin(); i_page != m_data.end(); ++i_page) { + fprintf(fout, "%s %08x:", m_name.c_str(), i_page->first); + i_page->second.print(format, fout); + } } -template void memory_space_impl::set_watch( addr_t addr, unsigned watchpoint ) -{ - m_watchpoints[watchpoint]=addr; +template +void memory_space_impl::set_watch(addr_t addr, unsigned watchpoint) { + m_watchpoints[watchpoint] = addr; } template class memory_space_impl<32>; template class memory_space_impl<64>; template class memory_space_impl<8192>; -template class memory_space_impl<16*1024>; +template class memory_space_impl<16 * 1024>; -void g_print_memory_space(memory_space *mem, const char *format = "%08x", FILE *fout = stdout) -{ - mem->print(format,fout); +void g_print_memory_space(memory_space *mem, const char *format = "%08x", + FILE *fout = stdout) { + mem->print(format, fout); } #ifdef UNIT_TEST -int main(int argc, char *argv[] ) -{ - int errors_found=0; - memory_space *mem = new memory_space_impl<32>("test",4); - // write address to [address] - for( mem_addr_t addr=0; addr < 16*1024; addr+=4) - mem->write(addr,4,&addr,NULL,NULL); - - for( mem_addr_t addr=0; addr < 16*1024; addr+=4) { - unsigned tmp=0; - mem->read(addr,4,&tmp); - if( tmp != addr ) { - errors_found=1; - printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, addr ); - } - } - - for( mem_addr_t addr=0; addr < 16*1024; addr+=1) { - unsigned char val = (addr + 128) % 256; - mem->write(addr,1,&val,NULL,NULL); - } - - for( mem_addr_t addr=0; addr < 16*1024; addr+=1) { - unsigned tmp=0; - mem->read(addr,1,&tmp); - unsigned char val = (addr + 128) % 256; - if( tmp != val ) { - errors_found=1; - printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, (unsigned)val ); - } - } - - if( errors_found ) { - printf("SUMMARY: ERRORS FOUND\n"); - } else { - printf("SUMMARY: UNIT TEST PASSED\n"); - } +int main(int argc, char *argv[]) { + int errors_found = 0; + memory_space *mem = new memory_space_impl<32>("test", 4); + // write address to [address] + for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 4) + mem->write(addr, 4, &addr, NULL, NULL); + + for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 4) { + unsigned tmp = 0; + mem->read(addr, 4, &tmp); + if (tmp != addr) { + errors_found = 1; + printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, addr); + } + } + + for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 1) { + unsigned char val = (addr + 128) % 256; + mem->write(addr, 1, &val, NULL, NULL); + } + + for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 1) { + unsigned tmp = 0; + mem->read(addr, 1, &tmp); + unsigned char val = (addr + 128) % 256; + if (tmp != val) { + errors_found = 1; + printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, + (unsigned)val); + } + } + + if (errors_found) { + printf("SUMMARY: ERRORS FOUND\n"); + } else { + printf("SUMMARY: UNIT TEST PASSED\n"); + } } #endif diff --git a/src/cuda-sim/memory.h b/src/cuda-sim/memory.h index ab588bc..31ad4df 100644 --- a/src/cuda-sim/memory.h +++ b/src/cuda-sim/memory.h @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -33,101 +35,97 @@ #include "../tr1_hash_map.h" #define mem_map tr1_hash_map #if tr1_hash_map_ismap == 1 - #define MEM_MAP_RESIZE(hash_size) +#define MEM_MAP_RESIZE(hash_size) #else - #define MEM_MAP_RESIZE(hash_size) (m_data.rehash(hash_size)) +#define MEM_MAP_RESIZE(hash_size) (m_data.rehash(hash_size)) #endif #include -#include #include -#include -#include #include +#include +#include +#include typedef address_type mem_addr_t; -#define MEM_BLOCK_SIZE (4*1024) - -template class mem_storage { -public: - mem_storage( const mem_storage &another ) - { - m_data = (unsigned char*)calloc(1,BSIZE); - memcpy(m_data,another.m_data,BSIZE); - } - mem_storage() - { - m_data = (unsigned char*)calloc(1,BSIZE); - } - ~mem_storage() - { - free(m_data); - } - - void write( unsigned offset, size_t length, const unsigned char *data ) - { - assert( offset + length <= BSIZE ); - memcpy(m_data+offset,data,length); - } - - void read( unsigned offset, size_t length, unsigned char *data ) const - { - assert( offset + length <= BSIZE ); - memcpy(data,m_data+offset,length); - } - - void print( const char *format, FILE *fout ) const - { - unsigned int *i_data = (unsigned int*)m_data; - for (int d = 0; d < (BSIZE / sizeof(unsigned int)); d++) { - if (d % 1 == 0) { - fprintf(fout, "\n"); - } - fprintf(fout, format, i_data[d]); - fprintf(fout, " "); +#define MEM_BLOCK_SIZE (4 * 1024) + +template +class mem_storage { + public: + mem_storage(const mem_storage &another) { + m_data = (unsigned char *)calloc(1, BSIZE); + memcpy(m_data, another.m_data, BSIZE); + } + mem_storage() { m_data = (unsigned char *)calloc(1, BSIZE); } + ~mem_storage() { free(m_data); } + + void write(unsigned offset, size_t length, const unsigned char *data) { + assert(offset + length <= BSIZE); + memcpy(m_data + offset, data, length); + } + + void read(unsigned offset, size_t length, unsigned char *data) const { + assert(offset + length <= BSIZE); + memcpy(data, m_data + offset, length); + } + + void print(const char *format, FILE *fout) const { + unsigned int *i_data = (unsigned int *)m_data; + for (int d = 0; d < (BSIZE / sizeof(unsigned int)); d++) { + if (d % 1 == 0) { + fprintf(fout, "\n"); } - fprintf(fout, "\n"); - fflush(fout); - } - -private: - unsigned m_nbytes; - unsigned char *m_data; + fprintf(fout, format, i_data[d]); + fprintf(fout, " "); + } + fprintf(fout, "\n"); + fflush(fout); + } + + private: + unsigned m_nbytes; + unsigned char *m_data; }; class ptx_thread_info; class ptx_instruction; -class memory_space -{ -public: - virtual ~memory_space() {} - virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ) = 0; - virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data ) = 0; - virtual void read( mem_addr_t addr, size_t length, void *data ) const = 0; - virtual void print( const char *format, FILE *fout ) const = 0; - virtual void set_watch( addr_t addr, unsigned watchpoint ) = 0; +class memory_space { + public: + virtual ~memory_space() {} + virtual void write(mem_addr_t addr, size_t length, const void *data, + ptx_thread_info *thd, const ptx_instruction *pI) = 0; + virtual void write_only(mem_addr_t index, mem_addr_t offset, size_t length, + const void *data) = 0; + virtual void read(mem_addr_t addr, size_t length, void *data) const = 0; + virtual void print(const char *format, FILE *fout) const = 0; + virtual void set_watch(addr_t addr, unsigned watchpoint) = 0; }; -template class memory_space_impl : public memory_space { -public: - memory_space_impl( std::string name, unsigned hash_size ); - - virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ); - virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data); - virtual void read( mem_addr_t addr, size_t length, void *data ) const; - virtual void print( const char *format, FILE *fout ) const; - - virtual void set_watch( addr_t addr, unsigned watchpoint ); - -private: - void read_single_block( mem_addr_t blk_idx, mem_addr_t addr, size_t length, void *data) const; - std::string m_name; - unsigned m_log2_block_size; - typedef mem_map > map_t; - map_t m_data; - std::map m_watchpoints; +template +class memory_space_impl : public memory_space { + public: + memory_space_impl(std::string name, unsigned hash_size); + + virtual void write(mem_addr_t addr, size_t length, const void *data, + ptx_thread_info *thd, const ptx_instruction *pI); + virtual void write_only(mem_addr_t index, mem_addr_t offset, size_t length, + const void *data); + virtual void read(mem_addr_t addr, size_t length, void *data) const; + virtual void print(const char *format, FILE *fout) const; + + virtual void set_watch(addr_t addr, unsigned watchpoint); + + private: + void read_single_block(mem_addr_t blk_idx, mem_addr_t addr, size_t length, + void *data) const; + std::string m_name; + unsigned m_log2_block_size; + typedef mem_map > map_t; + map_t m_data; + std::map m_watchpoints; }; #endif diff --git a/src/cuda-sim/opcodes.h b/src/cuda-sim/opcodes.h index 86d3b99..479b1ca 100644 --- a/src/cuda-sim/opcodes.h +++ b/src/cuda-sim/opcodes.h @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -29,49 +31,48 @@ #define opcodes_h_included enum opcode_t { -#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP, -#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP, +#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) OP, +#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) OP, #include "opcodes.def" - NUM_OPCODES + NUM_OPCODES #undef OP_DEF #undef OP_W_DEF }; enum special_regs { - CLOCK_REG, - HALFCLOCK_ID, - CLOCK64_REG, - CTAID_REG, - ENVREG_REG, - GRIDID_REG, - LANEID_REG, - LANEMASK_EQ_REG, - LANEMASK_LE_REG, - LANEMASK_LT_REG, - LANEMASK_GE_REG, - LANEMASK_GT_REG, - NCTAID_REG, - NTID_REG, - NSMID_REG, - NWARPID_REG, - PM_REG, - SMID_REG, - TID_REG, - WARPID_REG, - WARPSZ_REG + CLOCK_REG, + HALFCLOCK_ID, + CLOCK64_REG, + CTAID_REG, + ENVREG_REG, + GRIDID_REG, + LANEID_REG, + LANEMASK_EQ_REG, + LANEMASK_LE_REG, + LANEMASK_LT_REG, + LANEMASK_GE_REG, + LANEMASK_GT_REG, + NCTAID_REG, + NTID_REG, + NSMID_REG, + NWARPID_REG, + PM_REG, + SMID_REG, + TID_REG, + WARPID_REG, + WARPSZ_REG }; -enum wmma_type{ - LOAD_A, - LOAD_B, - LOAD_C, - STORE_D, - MMA, - ROW, - COL, - M16N16K16, - M32N8K16, - M8N32K16 - +enum wmma_type { + LOAD_A, + LOAD_B, + LOAD_C, + STORE_D, + MMA, + ROW, + COL, + M16N16K16, + M32N8K16, + M8N32K16 }; #endif diff --git a/src/cuda-sim/ptx-stats.cc b/src/cuda-sim/ptx-stats.cc index 22517df..2f16555 100644 --- a/src/cuda-sim/ptx-stats.cc +++ b/src/cuda-sim/ptx-stats.cc @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -25,248 +27,266 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#include "ptx_ir.h" -#include "ptx_sim.h" #include "ptx-stats.h" -#include "../option_parser.h" #include #include -#include "../tr1_hash_map.h" #include "../../libcuda/gpgpu_context.h" +#include "../option_parser.h" +#include "../tr1_hash_map.h" +#include "ptx_ir.h" +#include "ptx_sim.h" -void ptx_stats::ptx_file_line_stats_options(option_parser_t opp) -{ - option_parser_register(opp, "-enable_ptx_file_line_stats", OPT_BOOL, - &enable_ptx_file_line_stats, - "Turn on PTX source line statistic profiling. (1 = On)", "1"); - option_parser_register(opp, "-ptx_line_stats_filename", OPT_CSTR, - &ptx_line_stats_filename, - "Output file for PTX source line statistics.", "gpgpu_inst_stats.txt"); +void ptx_stats::ptx_file_line_stats_options(option_parser_t opp) { + option_parser_register( + opp, "-enable_ptx_file_line_stats", OPT_BOOL, &enable_ptx_file_line_stats, + "Turn on PTX source line statistic profiling. (1 = On)", "1"); + option_parser_register( + opp, "-ptx_line_stats_filename", OPT_CSTR, &ptx_line_stats_filename, + "Output file for PTX source line statistics.", "gpgpu_inst_stats.txt"); } // implementations // defining a PTX source line = filename + line number -class ptx_file_line -{ -public: - ptx_file_line(const char* s, int l) { - if( s == NULL ) - st = "NULL_NAME"; - else - st = s; - line = l; +class ptx_file_line { + public: + ptx_file_line(const char *s, int l) { + if (s == NULL) + st = "NULL_NAME"; + else + st = s; + line = l; + } + + bool operator<(const ptx_file_line &other) const { + if (st == other.st) { + if (line < other.line) + return true; + else + return false; + } else { + return st < other.st; } + } - bool operator<(const ptx_file_line &other) const { - if( st == other.st ) { - if( line < other.line ) - return true; - else - return false; - } else { - return st < other.st; - } - } + bool operator==(const ptx_file_line &other) const { + return (line == other.line) && (st == other.st); + } - bool operator==(const ptx_file_line &other) const { - return (line==other.line) && (st==other.st); - } - - std::string st; - unsigned line; + std::string st; + unsigned line; }; // holds all statistics collected for a singe PTX source line -class ptx_file_line_stats -{ -public: - ptx_file_line_stats() - : exec_count(0), latency(0), dram_traffic(0), - smem_n_way_bank_conflict_total(0), smem_warp_count(0), - gmem_n_access_total(0), gmem_warp_count(0), exposed_latency(0), - warp_divergence(0) - { } - - unsigned long exec_count; - unsigned long long latency; - unsigned long long dram_traffic; - unsigned long long smem_n_way_bank_conflict_total; // total number of banks accessed by this instruction - unsigned long smem_warp_count; // number of warps accessing shared memory - unsigned long long gmem_n_access_total; // number of uncoalesced access in total from this instruction - unsigned long gmem_warp_count; // number of warps causing these uncoalesced access - unsigned long long exposed_latency; // latency exposed as pipeline bubbles (attributed to this instruction) - unsigned long long warp_divergence; // number of warp divergence occured at this instruction +class ptx_file_line_stats { + public: + ptx_file_line_stats() + : exec_count(0), + latency(0), + dram_traffic(0), + smem_n_way_bank_conflict_total(0), + smem_warp_count(0), + gmem_n_access_total(0), + gmem_warp_count(0), + exposed_latency(0), + warp_divergence(0) {} + + unsigned long exec_count; + unsigned long long latency; + unsigned long long dram_traffic; + unsigned long long smem_n_way_bank_conflict_total; // total number of banks + // accessed by this + // instruction + unsigned long smem_warp_count; // number of warps accessing shared memory + unsigned long long gmem_n_access_total; // number of uncoalesced access in + // total from this instruction + unsigned long + gmem_warp_count; // number of warps causing these uncoalesced access + unsigned long long exposed_latency; // latency exposed as pipeline bubbles + // (attributed to this instruction) + unsigned long long + warp_divergence; // number of warp divergence occured at this instruction }; #if (tr1_hash_map_ismap == 1) -typedef tr1_hash_map ptx_file_line_stats_map_t; +typedef tr1_hash_map + ptx_file_line_stats_map_t; #else -struct hash_ptx_file_line -{ - std::size_t operator()(const ptx_file_line & pfline) const { - std::hash hash_line; - return hash_line(pfline.line); - } +struct hash_ptx_file_line { + std::size_t operator()(const ptx_file_line &pfline) const { + std::hash hash_line; + return hash_line(pfline.line); + } }; -typedef tr1_hash_map ptx_file_line_stats_map_t; +typedef tr1_hash_map + ptx_file_line_stats_map_t; #endif static ptx_file_line_stats_map_t ptx_file_line_stats_tracker; // output statistics to a file -void ptx_stats::ptx_file_line_stats_write_file() -{ - // check if stat collection is turned on - if (enable_ptx_file_line_stats == 0) return; - - ptx_file_line_stats_map_t::iterator it; - FILE * pfile; - - pfile = fopen(ptx_line_stats_filename, "w"); - fprintf(pfile,"kernel line : count latency dram_traffic smem_bk_conflicts smem_warp gmem_access_generated gmem_warp exposed_latency warp_divergence\n"); - for( it=ptx_file_line_stats_tracker.begin(); it != ptx_file_line_stats_tracker.end(); it++ ) { - fprintf(pfile, "%s %i : ", it->first.st.c_str(), it->first.line); - fprintf(pfile, "%lu ", it->second.exec_count); - fprintf(pfile, "%llu ", it->second.latency); - fprintf(pfile, "%llu ", it->second.dram_traffic); - fprintf(pfile, "%llu ", it->second.smem_n_way_bank_conflict_total); - fprintf(pfile, "%lu ", it->second.smem_warp_count); - fprintf(pfile, "%llu ", it->second.gmem_n_access_total); - fprintf(pfile, "%lu ", it->second.gmem_warp_count); - fprintf(pfile, "%llu ", it->second.exposed_latency); - fprintf(pfile, "%llu ", it->second.warp_divergence); - fprintf(pfile, "\n"); - } - fflush(pfile); - fclose(pfile); +void ptx_stats::ptx_file_line_stats_write_file() { + // check if stat collection is turned on + if (enable_ptx_file_line_stats == 0) return; + + ptx_file_line_stats_map_t::iterator it; + FILE *pfile; + + pfile = fopen(ptx_line_stats_filename, "w"); + fprintf(pfile, + "kernel line : count latency dram_traffic smem_bk_conflicts " + "smem_warp gmem_access_generated gmem_warp exposed_latency " + "warp_divergence\n"); + for (it = ptx_file_line_stats_tracker.begin(); + it != ptx_file_line_stats_tracker.end(); it++) { + fprintf(pfile, "%s %i : ", it->first.st.c_str(), it->first.line); + fprintf(pfile, "%lu ", it->second.exec_count); + fprintf(pfile, "%llu ", it->second.latency); + fprintf(pfile, "%llu ", it->second.dram_traffic); + fprintf(pfile, "%llu ", it->second.smem_n_way_bank_conflict_total); + fprintf(pfile, "%lu ", it->second.smem_warp_count); + fprintf(pfile, "%llu ", it->second.gmem_n_access_total); + fprintf(pfile, "%lu ", it->second.gmem_warp_count); + fprintf(pfile, "%llu ", it->second.exposed_latency); + fprintf(pfile, "%llu ", it->second.warp_divergence); + fprintf(pfile, "\n"); + } + fflush(pfile); + fclose(pfile); } // attribute one more execution count to this ptx instruction // counting the number of threads (not warps) executing this instruction -void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn) -{ - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].exec_count += 1; +void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn) { + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), + pInsn->source_line())] + .exec_count += 1; } // attribute pipeline latency to this ptx instruction (specified by the pc) -// pipeline latency is the number of cycles a warp with this instruction spent in the pipeline -void ptx_stats::ptx_file_line_stats_add_latency(unsigned pc, unsigned latency) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].latency += latency; +// pipeline latency is the number of cycles a warp with this instruction spent +// in the pipeline +void ptx_stats::ptx_file_line_stats_add_latency(unsigned pc, unsigned latency) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), + pInsn->source_line())] + .latency += latency; } // attribute dram traffic to this ptx instruction (specified by the pc) -// dram traffic is counted in number of requests -void ptx_stats::ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].dram_traffic += dram_traffic; +// dram traffic is counted in number of requests +void ptx_stats::ptx_file_line_stats_add_dram_traffic(unsigned pc, + unsigned dram_traffic) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), + pInsn->source_line())] + .dram_traffic += dram_traffic; } // attribute the number of shared memory access cycles to a ptx instruction -// counts both the number of warps doing shared memory access and the number of cycles involved -void ptx_stats::ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, unsigned n_way_bkconflict) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; - line_stats.smem_n_way_bank_conflict_total += n_way_bkconflict; - line_stats.smem_warp_count += 1; +// counts both the number of warps doing shared memory access and the number of +// cycles involved +void ptx_stats::ptx_file_line_stats_add_smem_bank_conflict( + unsigned pc, unsigned n_way_bkconflict) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( + pInsn->source_file(), pInsn->source_line())]; + line_stats.smem_n_way_bank_conflict_total += n_way_bkconflict; + line_stats.smem_warp_count += 1; } -// attribute a non-coalesced mem access to a ptx instruction -// counts both the number of warps causing this and the number of memory requests generated -void ptx_stats::ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; - line_stats.gmem_n_access_total += n_access; - line_stats.gmem_warp_count += 1; +// attribute a non-coalesced mem access to a ptx instruction +// counts both the number of warps causing this and the number of memory +// requests generated +void ptx_stats::ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, + unsigned n_access) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( + pInsn->source_file(), pInsn->source_line())]; + line_stats.gmem_n_access_total += n_access; + line_stats.gmem_warp_count += 1; } -// a class that tracks the inflight memory instructions of a shader core -// and attributes exposed latency to those instructions when signaled to do so -class ptx_inflight_memory_insn_tracker -{ -public: - typedef std::map insn_count_map; +// a class that tracks the inflight memory instructions of a shader core +// and attributes exposed latency to those instructions when signaled to do so +class ptx_inflight_memory_insn_tracker { + public: + typedef std::map insn_count_map; - void add_count(const ptx_instruction * pInsn, int count = 1) - { - ptx_inflight_memory_insns[pInsn] += count; - } + void add_count(const ptx_instruction *pInsn, int count = 1) { + ptx_inflight_memory_insns[pInsn] += count; + } - void sub_count(const ptx_instruction * pInsn, int count = 1) - { - insn_count_map::iterator i_insncount; - i_insncount = ptx_inflight_memory_insns.find(pInsn); + void sub_count(const ptx_instruction *pInsn, int count = 1) { + insn_count_map::iterator i_insncount; + i_insncount = ptx_inflight_memory_insns.find(pInsn); - assert(i_insncount != ptx_inflight_memory_insns.end()); + assert(i_insncount != ptx_inflight_memory_insns.end()); - i_insncount->second -= count; + i_insncount->second -= count; - if (i_insncount->second <= 0) { - ptx_inflight_memory_insns.erase(i_insncount); - } + if (i_insncount->second <= 0) { + ptx_inflight_memory_insns.erase(i_insncount); } - - void attribute_exposed_latency(int count = 1) - { - insn_count_map &exlat_insnmap = ptx_inflight_memory_insns; - insn_count_map::const_iterator i_exlatinsn; - - i_exlatinsn = exlat_insnmap.begin(); - for (; i_exlatinsn != exlat_insnmap.end(); ++i_exlatinsn) { - const ptx_instruction *pInsn = i_exlatinsn->first; - ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; - line_stats.exposed_latency += count; - } + } + + void attribute_exposed_latency(int count = 1) { + insn_count_map &exlat_insnmap = ptx_inflight_memory_insns; + insn_count_map::const_iterator i_exlatinsn; + + i_exlatinsn = exlat_insnmap.begin(); + for (; i_exlatinsn != exlat_insnmap.end(); ++i_exlatinsn) { + const ptx_instruction *pInsn = i_exlatinsn->first; + ptx_file_line_stats &line_stats = + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), + pInsn->source_line())]; + line_stats.exposed_latency += count; } + } - insn_count_map ptx_inflight_memory_insns; + insn_count_map ptx_inflight_memory_insns; }; static ptx_inflight_memory_insn_tracker *inflight_mem_tracker = NULL; -void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores) -{ - inflight_mem_tracker = new ptx_inflight_memory_insn_tracker[n_shader_cores]; +void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores) { + inflight_mem_tracker = new ptx_inflight_memory_insn_tracker[n_shader_cores]; } // add an inflight memory instruction -void ptx_stats::ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); +void ptx_stats::ptx_file_line_stats_add_inflight_memory_insn(int sc_id, + unsigned pc) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - inflight_mem_tracker[sc_id].add_count(pInsn); + inflight_mem_tracker[sc_id].add_count(pInsn); } // remove an inflight memory instruction -void ptx_stats::ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); +void ptx_stats::ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, + unsigned pc) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - inflight_mem_tracker[sc_id].sub_count(pInsn); + inflight_mem_tracker[sc_id].sub_count(pInsn); } -// attribute an empty cycle in the pipeline (exposed latency) to the ptx memory instructions in flight -void ptx_file_line_stats_commit_exposed_latency(int sc_id, int exposed_latency) -{ - assert(exposed_latency > 0); - inflight_mem_tracker[sc_id].attribute_exposed_latency(exposed_latency); +// attribute an empty cycle in the pipeline (exposed latency) to the ptx memory +// instructions in flight +void ptx_file_line_stats_commit_exposed_latency(int sc_id, + int exposed_latency) { + assert(exposed_latency > 0); + inflight_mem_tracker[sc_id].attribute_exposed_latency(exposed_latency); } // attribute the number of warp divergence to a ptx instruction -void ptx_stats::ptx_file_line_stats_add_warp_divergence(unsigned pc, unsigned n_way_divergence) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; - line_stats.warp_divergence += n_way_divergence; -} +void ptx_stats::ptx_file_line_stats_add_warp_divergence( + unsigned pc, unsigned n_way_divergence) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( + pInsn->source_file(), pInsn->source_line())]; + line_stats.warp_divergence += n_way_divergence; +} diff --git a/src/cuda-sim/ptx-stats.h b/src/cuda-sim/ptx-stats.h index 246b4ce..7745eba 100644 --- a/src/cuda-sim/ptx-stats.h +++ b/src/cuda-sim/ptx-stats.h @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -25,15 +27,14 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#pragma once +#pragma once #include "../option_parser.h" - #ifdef __cplusplus // stat collection interface to cuda-sim class ptx_instruction; -void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn); +void ptx_file_line_stats_add_exec_count(const ptx_instruction* pInsn); #endif // stat collection interface to gpgpu-sim @@ -41,28 +42,29 @@ void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn); void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores); void ptx_file_line_stats_commit_exposed_latency(int sc_id, int exposed_latency); - class gpgpu_context; class ptx_stats { - public: - ptx_stats(gpgpu_context* ctx) { - ptx_line_stats_filename = NULL; - gpgpu_ctx = ctx; - } - char * ptx_line_stats_filename; - bool enable_ptx_file_line_stats; - gpgpu_context* gpgpu_ctx; - // set options - void ptx_file_line_stats_options(option_parser_t opp); + public: + ptx_stats(gpgpu_context* ctx) { + ptx_line_stats_filename = NULL; + gpgpu_ctx = ctx; + } + char* ptx_line_stats_filename; + bool enable_ptx_file_line_stats; + gpgpu_context* gpgpu_ctx; + // set options + void ptx_file_line_stats_options(option_parser_t opp); - // output stats to a file - void ptx_file_line_stats_write_file(); - // stat collection interface to gpgpu-sim - void ptx_file_line_stats_add_latency(unsigned pc, unsigned latency); - void ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic); - void ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, unsigned n_way_bkconflict); - void ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access); - void ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc); - void ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc); - void ptx_file_line_stats_add_warp_divergence(unsigned pc, unsigned n_way_divergence); + // output stats to a file + void ptx_file_line_stats_write_file(); + // stat collection interface to gpgpu-sim + void ptx_file_line_stats_add_latency(unsigned pc, unsigned latency); + void ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic); + void ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, + unsigned n_way_bkconflict); + void ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access); + void ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc); + void ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc); + void ptx_file_line_stats_add_warp_divergence(unsigned pc, + unsigned n_way_divergence); }; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index d8943d2..55afb51 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -1,5 +1,5 @@ // Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda, Wilson W.L. Fung, -// George L. Yuan +// George L. Yuan // The University of British Columbia // All rights reserved. // @@ -8,14 +8,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -26,1152 +28,1244 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#include "ptx_parser.h" #include "ptx_ir.h" -typedef void * yyscan_t; -#include "ptx.tab.h" -#include "opcodes.h" +#include "ptx_parser.h" +typedef void *yyscan_t; +#include #include #include -#include -#include #include +#include #include "assert.h" +#include "opcodes.h" +#include "ptx.tab.h" -#include "cuda-sim.h" #include "../../libcuda/gpgpu_context.h" +#include "cuda-sim.h" #define STR_SIZE 1024 -const ptx_instruction* gpgpu_context::pc_to_instruction(unsigned pc) -{ - if( pc < s_g_pc_to_insn.size() ) - return s_g_pc_to_insn[pc]; - else - return NULL; -} - -unsigned symbol::get_uid() -{ - unsigned result = (gpgpu_ctx->symbol_sm_next_uid)++; - return result; +const ptx_instruction *gpgpu_context::pc_to_instruction(unsigned pc) { + if (pc < s_g_pc_to_insn.size()) + return s_g_pc_to_insn[pc]; + else + return NULL; } -void symbol::add_initializer( const std::list &init ) -{ - m_initializer = init; -} - -void symbol::print_info(FILE *fp) const -{ - fprintf(fp,"uid:%u, decl:%s, type:%p, ", m_uid, m_decl_location.c_str(), m_type ); - if( m_address_valid ) - fprintf(fp,"
, "); - if( m_is_label ) - fprintf(fp," is_label "); - if( m_is_shared ) - fprintf(fp," is_shared "); - if( m_is_const ) - fprintf(fp," is_const "); - if( m_is_global ) - fprintf(fp," is_global "); - if( m_is_local ) - fprintf(fp," is_local "); - if( m_is_tex ) - fprintf(fp," is_tex "); - if( m_is_func_addr ) - fprintf(fp," is_func_addr "); - if( m_function ) - fprintf(fp," %p ", m_function ); +unsigned symbol::get_uid() { + unsigned result = (gpgpu_ctx->symbol_sm_next_uid)++; + return result; } -symbol_table::symbol_table() -{ - assert(0); +void symbol::add_initializer(const std::list &init) { + m_initializer = init; } -symbol_table::symbol_table( const char *scope_name, unsigned entry_point, symbol_table *parent, gpgpu_context* ctx ) -{ - gpgpu_ctx = ctx; - m_scope_name = std::string(scope_name); - m_reg_allocator=0; - m_shared_next = 0; - m_const_next = 0; - m_global_next = 0x100; - m_local_next = 0; - m_tex_next = 0; - - //Jin: handle instruction group for cdp - m_inst_group_id = 0; - - m_parent = parent; - if ( m_parent ) { - m_shared_next = m_parent->m_shared_next; - m_global_next = m_parent->m_global_next; - } +void symbol::print_info(FILE *fp) const { + fprintf(fp, "uid:%u, decl:%s, type:%p, ", m_uid, m_decl_location.c_str(), + m_type); + if (m_address_valid) fprintf(fp, "
, "); + if (m_is_label) fprintf(fp, " is_label "); + if (m_is_shared) fprintf(fp, " is_shared "); + if (m_is_const) fprintf(fp, " is_const "); + if (m_is_global) fprintf(fp, " is_global "); + if (m_is_local) fprintf(fp, " is_local "); + if (m_is_tex) fprintf(fp, " is_tex "); + if (m_is_func_addr) fprintf(fp, " is_func_addr "); + if (m_function) fprintf(fp, " %p ", m_function); } -void symbol_table::set_name( const char *name ) -{ - m_scope_name = std::string(name); -} +symbol_table::symbol_table() { assert(0); } -const ptx_version &symbol_table::get_ptx_version() const -{ - if( m_parent == NULL ) return m_ptx_version; - else return m_parent->get_ptx_version(); -} +symbol_table::symbol_table(const char *scope_name, unsigned entry_point, + symbol_table *parent, gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_scope_name = std::string(scope_name); + m_reg_allocator = 0; + m_shared_next = 0; + m_const_next = 0; + m_global_next = 0x100; + m_local_next = 0; + m_tex_next = 0; + + // Jin: handle instruction group for cdp + m_inst_group_id = 0; -unsigned symbol_table::get_sm_target() const -{ - if( m_parent == NULL ) - return m_ptx_version.target(); - else return m_parent->get_sm_target(); + m_parent = parent; + if (m_parent) { + m_shared_next = m_parent->m_shared_next; + m_global_next = m_parent->m_global_next; + } } -void symbol_table::set_ptx_version( float ver, unsigned ext ) -{ - m_ptx_version = ptx_version(ver,ext); +void symbol_table::set_name(const char *name) { + m_scope_name = std::string(name); } -void symbol_table::set_sm_target( const char *target, const char *ext, const char *ext2 ) -{ - m_ptx_version.set_target(target,ext,ext2); +const ptx_version &symbol_table::get_ptx_version() const { + if (m_parent == NULL) + return m_ptx_version; + else + return m_parent->get_ptx_version(); } -symbol *symbol_table::lookup( const char *identifier ) -{ - std::string key(identifier); - std::map::iterator i = m_symbols.find(key); - if ( i != m_symbols.end() ) { - return i->second; - } - if ( m_parent ) { - return m_parent->lookup(identifier); - } - return NULL; +unsigned symbol_table::get_sm_target() const { + if (m_parent == NULL) + return m_ptx_version.target(); + else + return m_parent->get_sm_target(); } -symbol *symbol_table::add_variable( const char *identifier, const type_info *type, unsigned size, const char *filename, unsigned line ) -{ - char buf[1024]; - std::string key(identifier); - assert( m_symbols.find(key) == m_symbols.end() ); - snprintf(buf,1024,"%s:%u",filename,line); - symbol *s = new symbol(identifier,type,buf,size,gpgpu_ctx); - m_symbols[ key ] = s; - - if ( type != NULL && type->get_key().is_global() ) { - m_globals.push_back(s); - } - if ( type != NULL && type->get_key().is_const() ) { - m_consts.push_back(s); - } - - return s; +void symbol_table::set_ptx_version(float ver, unsigned ext) { + m_ptx_version = ptx_version(ver, ext); } - -void symbol_table::add_function( function_info *func, const char *filename, unsigned linenumber ) -{ - std::map::iterator i = m_symbols.find( func->get_name() ); - if( i != m_symbols.end() ) - return; - char buf[1024]; - snprintf(buf,1024,"%s:%u",filename,linenumber); - type_info *type = add_type( func ); - symbol *s = new symbol(func->get_name().c_str(),type,buf,0,gpgpu_ctx); - s->set_function(func); - m_symbols[ func->get_name() ] = s; -} - -//Jin: handle instruction group for cdp -symbol_table* symbol_table::start_inst_group() { - char inst_group_name[4096]; - snprintf(inst_group_name, 4096, "%s_inst_group_%u", m_scope_name.c_str(), m_inst_group_id); - - //previous added - assert(m_inst_group_symtab.find(std::string(inst_group_name)) == m_inst_group_symtab.end()); - symbol_table *sym_table = new symbol_table(inst_group_name, 3/*inst group*/, this, gpgpu_ctx ); - - sym_table->m_global_next = m_global_next; - sym_table->m_shared_next = m_shared_next; - sym_table->m_local_next = m_local_next; - sym_table->m_reg_allocator = m_reg_allocator; - sym_table->m_tex_next = m_tex_next; - sym_table->m_const_next = m_const_next; - - m_inst_group_symtab[std::string(inst_group_name)] = sym_table; - - return sym_table; -} - -symbol_table * symbol_table::end_inst_group() { - symbol_table * sym_table = m_parent; - - sym_table->m_global_next = m_global_next; - sym_table->m_shared_next = m_shared_next; - sym_table->m_local_next = m_local_next; - sym_table->m_reg_allocator = m_reg_allocator; - sym_table->m_tex_next = m_tex_next; - sym_table->m_const_next = m_const_next; - sym_table->m_inst_group_id++; - - return sym_table; -} - -void register_ptx_function( const char *name, function_info *impl ); // either libcuda or libopencl - -bool symbol_table::add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **sym_table ) -{ - std::string key = std::string(name); - bool prior_decl = false; - if( m_function_info_lookup.find(key) != m_function_info_lookup.end() ) { - *func_info = m_function_info_lookup[key]; - prior_decl = true; - } else { - *func_info = new function_info(entry_point, gpgpu_ctx); - (*func_info)->set_name(name); - (*func_info)->set_maxnt_id(0); - m_function_info_lookup[key] = *func_info; - } - - if( m_function_symtab_lookup.find(key) != m_function_symtab_lookup.end() ) { - assert( prior_decl ); - *sym_table = m_function_symtab_lookup[key]; - } else { - assert( !prior_decl ); - *sym_table = new symbol_table( "", entry_point, this, gpgpu_ctx ); - - // Initial setup code to support a register represented as "_". - // This register is used when an instruction operand is - // not read or written. However, the parser must recognize it - // as a legitimate register but we do not want to pass - // it to the micro-architectural register to the performance simulator. - // For this purpose we add a symbol to the symbol table but - // mark it as a non_arch_reg so it does not effect the performance sim. - type_info_key null_key( reg_space, 0, 0, 0, 0, 0 ); - null_key.set_is_non_arch_reg(); - // First param is null - which is bad. - // However, the first parameter is actually unread in the constructor... - // TODO - remove the symbol_table* from type_info - type_info* null_type_info = new type_info( NULL, null_key ); - symbol *null_reg = (*sym_table)->add_variable( "_", null_type_info, 0, "", 0 ); - null_reg->set_regno(0, 0); - - (*sym_table)->set_name(name); - (*func_info)->set_symtab(*sym_table); - m_function_symtab_lookup[key] = *sym_table; - assert( (*func_info)->get_symtab() == *sym_table ); - register_ptx_function(name,*func_info); - } - return prior_decl; -} - -function_info *symbol_table::lookup_function( std::string name ) -{ - std::string key = std::string(name); - std::map::iterator it = m_function_info_lookup.find(key); - assert ( it != m_function_info_lookup.end() ); - return it->second; -} - -type_info *symbol_table::add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ) -{ - if( space_spec == param_space_unclassified ) - space_spec = param_space_local; - type_info_key t(space_spec,scalar_type_spec,vector_spec,alignment_spec,extern_spec,0); - type_info *pt; - pt = new type_info(this,t); - return pt; + +void symbol_table::set_sm_target(const char *target, const char *ext, + const char *ext2) { + m_ptx_version.set_target(target, ext, ext2); +} + +symbol *symbol_table::lookup(const char *identifier) { + std::string key(identifier); + std::map::iterator i = m_symbols.find(key); + if (i != m_symbols.end()) { + return i->second; + } + if (m_parent) { + return m_parent->lookup(identifier); + } + return NULL; +} + +symbol *symbol_table::add_variable(const char *identifier, + const type_info *type, unsigned size, + const char *filename, unsigned line) { + char buf[1024]; + std::string key(identifier); + assert(m_symbols.find(key) == m_symbols.end()); + snprintf(buf, 1024, "%s:%u", filename, line); + symbol *s = new symbol(identifier, type, buf, size, gpgpu_ctx); + m_symbols[key] = s; + + if (type != NULL && type->get_key().is_global()) { + m_globals.push_back(s); + } + if (type != NULL && type->get_key().is_const()) { + m_consts.push_back(s); + } + + return s; +} + +void symbol_table::add_function(function_info *func, const char *filename, + unsigned linenumber) { + std::map::iterator i = + m_symbols.find(func->get_name()); + if (i != m_symbols.end()) return; + char buf[1024]; + snprintf(buf, 1024, "%s:%u", filename, linenumber); + type_info *type = add_type(func); + symbol *s = new symbol(func->get_name().c_str(), type, buf, 0, gpgpu_ctx); + s->set_function(func); + m_symbols[func->get_name()] = s; +} + +// Jin: handle instruction group for cdp +symbol_table *symbol_table::start_inst_group() { + char inst_group_name[4096]; + snprintf(inst_group_name, 4096, "%s_inst_group_%u", m_scope_name.c_str(), + m_inst_group_id); + + // previous added + assert(m_inst_group_symtab.find(std::string(inst_group_name)) == + m_inst_group_symtab.end()); + symbol_table *sym_table = + new symbol_table(inst_group_name, 3 /*inst group*/, this, gpgpu_ctx); + + sym_table->m_global_next = m_global_next; + sym_table->m_shared_next = m_shared_next; + sym_table->m_local_next = m_local_next; + sym_table->m_reg_allocator = m_reg_allocator; + sym_table->m_tex_next = m_tex_next; + sym_table->m_const_next = m_const_next; + + m_inst_group_symtab[std::string(inst_group_name)] = sym_table; + + return sym_table; +} + +symbol_table *symbol_table::end_inst_group() { + symbol_table *sym_table = m_parent; + + sym_table->m_global_next = m_global_next; + sym_table->m_shared_next = m_shared_next; + sym_table->m_local_next = m_local_next; + sym_table->m_reg_allocator = m_reg_allocator; + sym_table->m_tex_next = m_tex_next; + sym_table->m_const_next = m_const_next; + sym_table->m_inst_group_id++; + + return sym_table; +} + +void register_ptx_function(const char *name, + function_info *impl); // either libcuda or libopencl + +bool symbol_table::add_function_decl(const char *name, int entry_point, + function_info **func_info, + symbol_table **sym_table) { + std::string key = std::string(name); + bool prior_decl = false; + if (m_function_info_lookup.find(key) != m_function_info_lookup.end()) { + *func_info = m_function_info_lookup[key]; + prior_decl = true; + } else { + *func_info = new function_info(entry_point, gpgpu_ctx); + (*func_info)->set_name(name); + (*func_info)->set_maxnt_id(0); + m_function_info_lookup[key] = *func_info; + } + + if (m_function_symtab_lookup.find(key) != m_function_symtab_lookup.end()) { + assert(prior_decl); + *sym_table = m_function_symtab_lookup[key]; + } else { + assert(!prior_decl); + *sym_table = new symbol_table("", entry_point, this, gpgpu_ctx); + + // Initial setup code to support a register represented as "_". + // This register is used when an instruction operand is + // not read or written. However, the parser must recognize it + // as a legitimate register but we do not want to pass + // it to the micro-architectural register to the performance simulator. + // For this purpose we add a symbol to the symbol table but + // mark it as a non_arch_reg so it does not effect the performance sim. + type_info_key null_key(reg_space, 0, 0, 0, 0, 0); + null_key.set_is_non_arch_reg(); + // First param is null - which is bad. + // However, the first parameter is actually unread in the constructor... + // TODO - remove the symbol_table* from type_info + type_info *null_type_info = new type_info(NULL, null_key); + symbol *null_reg = + (*sym_table)->add_variable("_", null_type_info, 0, "", 0); + null_reg->set_regno(0, 0); + + (*sym_table)->set_name(name); + (*func_info)->set_symtab(*sym_table); + m_function_symtab_lookup[key] = *sym_table; + assert((*func_info)->get_symtab() == *sym_table); + register_ptx_function(name, *func_info); + } + return prior_decl; +} + +function_info *symbol_table::lookup_function(std::string name) { + std::string key = std::string(name); + std::map::iterator it = + m_function_info_lookup.find(key); + assert(it != m_function_info_lookup.end()); + return it->second; +} + +type_info *symbol_table::add_type(memory_space_t space_spec, + int scalar_type_spec, int vector_spec, + int alignment_spec, int extern_spec) { + if (space_spec == param_space_unclassified) space_spec = param_space_local; + type_info_key t(space_spec, scalar_type_spec, vector_spec, alignment_spec, + extern_spec, 0); + type_info *pt; + pt = new type_info(this, t); + return pt; +} + +type_info *symbol_table::add_type(function_info *func) { + type_info_key t; + type_info *pt; + t.set_is_func(); + pt = new type_info(this, t); + return pt; +} + +type_info *symbol_table::get_array_type(type_info *base_type, + unsigned array_dim) { + type_info_key t = base_type->get_key(); + t.set_array_dim(array_dim); + type_info *pt = new type_info(this, t); + // Where else is m_types being used? As of now, I dont find any use of it and + // causing seg fault. So disabling m_types. + // TODO: find where m_types can be used in future and solve the seg fault. + // pt = m_types[t] = new type_info(this,t); + return pt; +} + +void symbol_table::set_label_address(const symbol *label, unsigned addr) { + std::map::iterator i = m_symbols.find(label->name()); + assert(i != m_symbols.end()); + symbol *s = i->second; + s->set_label_address(addr); +} + +void symbol_table::dump() { + printf("\n\n"); + printf("Symbol table for \"%s\":\n", m_scope_name.c_str()); + std::map::iterator i; + for (i = m_symbols.begin(); i != m_symbols.end(); i++) { + printf("%30s : ", i->first.c_str()); + if (i->second) + i->second->print_info(stdout); + else + printf(" "); + printf("\n"); + } + printf("\n"); } -type_info *symbol_table::add_type( function_info *func ) -{ - type_info_key t; - type_info *pt; - t.set_is_func(); - pt = new type_info(this,t); - return pt; +unsigned operand_info::get_uid() { + unsigned result = (gpgpu_ctx->operand_info_sm_next_uid)++; + return result; } -type_info *symbol_table::get_array_type( type_info *base_type, unsigned array_dim ) -{ - type_info_key t = base_type->get_key(); - t.set_array_dim(array_dim); - type_info *pt = new type_info(this,t); - //Where else is m_types being used? As of now, I dont find any use of it and causing seg fault. So disabling m_types. - //TODO: find where m_types can be used in future and solve the seg fault. - //pt = m_types[t] = new type_info(this,t); - return pt; +std::list::iterator +function_info::find_next_real_instruction( + std::list::iterator i) { + while ((i != m_instructions.end()) && (*i)->is_label()) i++; + return i; } -void symbol_table::set_label_address( const symbol *label, unsigned addr ) -{ - std::map::iterator i=m_symbols.find(label->name()); - assert( i != m_symbols.end() ); - symbol *s = i->second; - s->set_label_address(addr); -} +void function_info::create_basic_blocks() { + std::list leaders; + std::list::iterator i, l; -void symbol_table::dump() -{ - printf("\n\n"); - printf("Symbol table for \"%s\":\n", m_scope_name.c_str() ); - std::map::iterator i; - for( i=m_symbols.begin(); i!=m_symbols.end(); i++ ) { - printf("%30s : ", i->first.c_str() ); - if( i->second ) - i->second->print_info(stdout); - else - printf(" "); - printf("\n"); - } - printf("\n"); -} - -unsigned operand_info::get_uid() -{ - unsigned result = (gpgpu_ctx->operand_info_sm_next_uid)++; - return result; -} - -std::list::iterator function_info::find_next_real_instruction( std::list::iterator i) -{ - while( (i != m_instructions.end()) && (*i)->is_label() ) - i++; - return i; -} - -void function_info::create_basic_blocks() -{ - std::list leaders; - std::list::iterator i, l; - - // first instruction is a leader - i=m_instructions.begin(); - leaders.push_back(*i); - i++; - while( i!=m_instructions.end() ) { - ptx_instruction *pI = *i; - if( pI->is_label() ) { - leaders.push_back(pI); - i = find_next_real_instruction(++i); - } else { - switch( pI->get_opcode() ) { - case BRA_OP: case RET_OP: case EXIT_OP: case RETP_OP: case BREAK_OP: + // first instruction is a leader + i = m_instructions.begin(); + leaders.push_back(*i); + i++; + while (i != m_instructions.end()) { + ptx_instruction *pI = *i; + if (pI->is_label()) { + leaders.push_back(pI); + i = find_next_real_instruction(++i); + } else { + switch (pI->get_opcode()) { + case BRA_OP: + case RET_OP: + case EXIT_OP: + case RETP_OP: + case BREAK_OP: + i++; + if (i != m_instructions.end()) leaders.push_back(*i); + i = find_next_real_instruction(i); + break; + case CALL_OP: + case CALLP_OP: + if (pI->has_pred()) { + printf("GPGPU-Sim PTX: Warning found predicated call\n"); i++; - if( i != m_instructions.end() ) - leaders.push_back(*i); + if (i != m_instructions.end()) leaders.push_back(*i); i = find_next_real_instruction(i); - break; - case CALL_OP: case CALLP_OP: - if( pI->has_pred() ) { - printf("GPGPU-Sim PTX: Warning found predicated call\n"); - i++; - if( i != m_instructions.end() ) - leaders.push_back(*i); - i = find_next_real_instruction(i); - } else i++; - break; - default: + } else i++; - } - } - } - - if( leaders.empty() ) { - printf("GPGPU-Sim PTX: Function \'%s\' has no basic blocks\n", m_name.c_str()); - return; - } - - unsigned bb_id = 0; - l=leaders.begin(); - i=m_instructions.begin(); - m_basic_blocks.push_back( new basic_block_t(bb_id++,*find_next_real_instruction(i),NULL,1,0) ); - ptx_instruction *last_real_inst=*(l++); - - for( ; i!=m_instructions.end(); i++ ) { - ptx_instruction *pI = *i; - if( l != leaders.end() && *i == *l ) { - // found start of next basic block - m_basic_blocks.back()->ptx_end = last_real_inst; - if( find_next_real_instruction(i) != m_instructions.end() ) { // if not bogus trailing label - m_basic_blocks.push_back( new basic_block_t(bb_id++,*find_next_real_instruction(i),NULL,0,0) ); - last_real_inst = *find_next_real_instruction(i); - } - // start search for next leader - l++; + break; + default: + i++; } - pI->assign_bb( m_basic_blocks.back() ); - if( !pI->is_label() ) last_real_inst = pI; - } - m_basic_blocks.back()->ptx_end = last_real_inst; - m_basic_blocks.push_back( /*exit basic block*/ new basic_block_t(bb_id,NULL,NULL,0,1) ); -} - -void function_info::print_basic_blocks() -{ - printf("Printing basic blocks for function \'%s\':\n", m_name.c_str() ); - std::list::iterator ptx_itr; - unsigned last_bb=0; - for (ptx_itr = m_instructions.begin();ptx_itr != m_instructions.end(); ptx_itr++) { - if( (*ptx_itr)->get_bb() ) { - if( (*ptx_itr)->get_bb()->bb_id != last_bb ) { - printf("\n"); - last_bb = (*ptx_itr)->get_bb()->bb_id; - } - printf("bb_%02u\t: ", (*ptx_itr)->get_bb()->bb_id); - (*ptx_itr)->print_insn(); - printf("\n"); + } + } + + if (leaders.empty()) { + printf("GPGPU-Sim PTX: Function \'%s\' has no basic blocks\n", + m_name.c_str()); + return; + } + + unsigned bb_id = 0; + l = leaders.begin(); + i = m_instructions.begin(); + m_basic_blocks.push_back( + new basic_block_t(bb_id++, *find_next_real_instruction(i), NULL, 1, 0)); + ptx_instruction *last_real_inst = *(l++); + + for (; i != m_instructions.end(); i++) { + ptx_instruction *pI = *i; + if (l != leaders.end() && *i == *l) { + // found start of next basic block + m_basic_blocks.back()->ptx_end = last_real_inst; + if (find_next_real_instruction(i) != + m_instructions.end()) { // if not bogus trailing label + m_basic_blocks.push_back(new basic_block_t( + bb_id++, *find_next_real_instruction(i), NULL, 0, 0)); + last_real_inst = *find_next_real_instruction(i); } - } - printf("\nSummary of basic blocks for \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - printf("bb_%02u\t:", (*bb_itr)->bb_id); - if ((*bb_itr)->ptx_begin) - printf(" first: %s\t", ((*bb_itr)->ptx_begin)->get_opcode_cstr()); - else printf(" first: NULL\t"); - if ((*bb_itr)->ptx_end) { - printf(" last: %s\t", ((*bb_itr)->ptx_end)->get_opcode_cstr()); - } else printf(" last: NULL\t"); + // start search for next leader + l++; + } + pI->assign_bb(m_basic_blocks.back()); + if (!pI->is_label()) last_real_inst = pI; + } + m_basic_blocks.back()->ptx_end = last_real_inst; + m_basic_blocks.push_back( + /*exit basic block*/ new basic_block_t(bb_id, NULL, NULL, 0, 1)); +} + +void function_info::print_basic_blocks() { + printf("Printing basic blocks for function \'%s\':\n", m_name.c_str()); + std::list::iterator ptx_itr; + unsigned last_bb = 0; + for (ptx_itr = m_instructions.begin(); ptx_itr != m_instructions.end(); + ptx_itr++) { + if ((*ptx_itr)->get_bb()) { + if ((*ptx_itr)->get_bb()->bb_id != last_bb) { + printf("\n"); + last_bb = (*ptx_itr)->get_bb()->bb_id; + } + printf("bb_%02u\t: ", (*ptx_itr)->get_bb()->bb_id); + (*ptx_itr)->print_insn(); printf("\n"); - } - printf("\n"); -} - -void function_info::print_basic_block_links() -{ - printf("Printing basic blocks links for function \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - printf("ID: %d\t:", (*bb_itr)->bb_id); - if ( !(*bb_itr)->predecessor_ids.empty() ) { - printf("Predecessors:"); - std::set::iterator p; - for (p= (*bb_itr)->predecessor_ids.begin();p != (*bb_itr)->predecessor_ids.end();p++) { - printf(" %d", *p); - } - printf("\t"); + } + } + printf("\nSummary of basic blocks for \'%s\':\n", m_name.c_str()); + std::vector::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + printf("bb_%02u\t:", (*bb_itr)->bb_id); + if ((*bb_itr)->ptx_begin) + printf(" first: %s\t", ((*bb_itr)->ptx_begin)->get_opcode_cstr()); + else + printf(" first: NULL\t"); + if ((*bb_itr)->ptx_end) { + printf(" last: %s\t", ((*bb_itr)->ptx_end)->get_opcode_cstr()); + } else + printf(" last: NULL\t"); + printf("\n"); + } + printf("\n"); +} + +void function_info::print_basic_block_links() { + printf("Printing basic blocks links for function \'%s\':\n", m_name.c_str()); + std::vector::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + printf("ID: %d\t:", (*bb_itr)->bb_id); + if (!(*bb_itr)->predecessor_ids.empty()) { + printf("Predecessors:"); + std::set::iterator p; + for (p = (*bb_itr)->predecessor_ids.begin(); + p != (*bb_itr)->predecessor_ids.end(); p++) { + printf(" %d", *p); } - if ( !(*bb_itr)->successor_ids.empty() ) { - printf("Successors:"); - std::set::iterator s; - for (s= (*bb_itr)->successor_ids.begin();s != (*bb_itr)->successor_ids.end();s++) { - printf(" %d", *s); - } + printf("\t"); + } + if (!(*bb_itr)->successor_ids.empty()) { + printf("Successors:"); + std::set::iterator s; + for (s = (*bb_itr)->successor_ids.begin(); + s != (*bb_itr)->successor_ids.end(); s++) { + printf(" %d", *s); } - printf("\n"); - } + } + printf("\n"); + } } -operand_info* function_info::find_break_target( ptx_instruction * p_break_insn ) //find the target of a break instruction +operand_info *function_info::find_break_target( + ptx_instruction *p_break_insn) // find the target of a break instruction { - const basic_block_t *break_bb = p_break_insn->get_bb(); - // go through the dominator tree - for(const basic_block_t *p_bb = break_bb; - p_bb->immediatedominator_id != -1; - p_bb = m_basic_blocks[p_bb->immediatedominator_id]) - { - // reverse search through instructions in basic block for breakaddr instruction - unsigned insn_addr = p_bb->ptx_end->get_m_instr_mem_index(); - while (insn_addr >= p_bb->ptx_begin->get_m_instr_mem_index()) { - ptx_instruction *pI = m_instr_mem[insn_addr]; - insn_addr -= 1; - if (pI == NULL) continue; // temporary solution for variable size instructions - if (pI->get_opcode() == BREAKADDR_OP) { - return &(pI->dst()); - } + const basic_block_t *break_bb = p_break_insn->get_bb(); + // go through the dominator tree + for (const basic_block_t *p_bb = break_bb; p_bb->immediatedominator_id != -1; + p_bb = m_basic_blocks[p_bb->immediatedominator_id]) { + // reverse search through instructions in basic block for breakaddr + // instruction + unsigned insn_addr = p_bb->ptx_end->get_m_instr_mem_index(); + while (insn_addr >= p_bb->ptx_begin->get_m_instr_mem_index()) { + ptx_instruction *pI = m_instr_mem[insn_addr]; + insn_addr -= 1; + if (pI == NULL) + continue; // temporary solution for variable size instructions + if (pI->get_opcode() == BREAKADDR_OP) { + return &(pI->dst()); } - } + } + } - assert(0); + assert(0); - // lazy fallback: just traverse backwards? - for (int insn_addr = p_break_insn->get_m_instr_mem_index(); - insn_addr >= 0; insn_addr--) - { - ptx_instruction *pI = m_instr_mem[insn_addr]; - if (pI->get_opcode() == BREAKADDR_OP) { - return &(pI->dst()); - } - } + // lazy fallback: just traverse backwards? + for (int insn_addr = p_break_insn->get_m_instr_mem_index(); insn_addr >= 0; + insn_addr--) { + ptx_instruction *pI = m_instr_mem[insn_addr]; + if (pI->get_opcode() == BREAKADDR_OP) { + return &(pI->dst()); + } + } - return NULL; + return NULL; } -void function_info::connect_basic_blocks( ) //iterate across m_basic_blocks of function, connecting basic blocks together +void function_info::connect_basic_blocks() // iterate across m_basic_blocks of + // function, connecting basic blocks + // together { - std::vector::iterator bb_itr; - std::vector::iterator bb_target_itr; - basic_block_t* exit_bb = m_basic_blocks.back(); - - //start from first basic block, which we know is the entry point - bb_itr = m_basic_blocks.begin(); - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - ptx_instruction *pI = (*bb_itr)->ptx_end; - if ((*bb_itr)->is_exit) //reached last basic block, no successors to link - continue; - if (pI->get_opcode() == RETP_OP || pI->get_opcode() == RET_OP || pI->get_opcode() == EXIT_OP ) { - (*bb_itr)->successor_ids.insert(exit_bb->bb_id); - exit_bb->predecessor_ids.insert((*bb_itr)->bb_id); - if( pI->has_pred() ) { - printf("GPGPU-Sim PTX: Warning detected predicated return/exit.\n"); - // if predicated, add link to next block - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - if( next_addr < m_instr_mem_size && m_instr_mem[next_addr] ) { - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - (*bb_itr)->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } - } - continue; - } else if (pI->get_opcode() == BRA_OP) { - //find successor and link that basic_block to this one - operand_info &target = pI->dst(); //get operand, e.g. target name - unsigned addr = labels[ target.name() ]; - ptx_instruction *target_pI = m_instr_mem[addr]; - basic_block_t *target_bb = target_pI->get_bb(); - (*bb_itr)->successor_ids.insert(target_bb->bb_id); - target_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } - - if ( !(pI->get_opcode()==BRA_OP && (!pI->has_pred())) ) { - // if basic block does not end in an unpredicated branch, - // then next basic block is also successor - // (this is better than testing for .uni) - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - (*bb_itr)->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } else - assert(pI->get_opcode() == BRA_OP); - } -} -bool function_info::connect_break_targets() //connecting break instructions with proper targets -{ - std::vector::iterator bb_itr; - std::vector::iterator bb_target_itr; - bool modified = false; - - //start from first basic block, which we know is the entry point - bb_itr = m_basic_blocks.begin(); - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - basic_block_t *p_bb = *bb_itr; - ptx_instruction *pI = p_bb->ptx_end; - if (p_bb->is_exit) //reached last basic block, no successors to link - continue; - if (pI->get_opcode() == BREAK_OP) { - // backup existing successor_ids for stability check - std::set orig_successor_ids = p_bb->successor_ids; - - // erase the previous linkage with old successors - for(std::set::iterator succ_ids = p_bb->successor_ids.begin(); succ_ids != p_bb->successor_ids.end(); ++succ_ids) { - basic_block_t *successor_bb = m_basic_blocks[*succ_ids]; - successor_bb->predecessor_ids.erase(p_bb->bb_id); - } - p_bb->successor_ids.clear(); - - //find successor and link that basic_block to this one - //successor of a break is set by an preceeding breakaddr instruction - operand_info *target = find_break_target(pI); - unsigned addr = labels[ target->name() ]; - ptx_instruction *target_pI = m_instr_mem[addr]; - basic_block_t *target_bb = target_pI->get_bb(); - p_bb->successor_ids.insert(target_bb->bb_id); - target_bb->predecessor_ids.insert(p_bb->bb_id); - - if (pI->has_pred()) { - // predicated break - add link to next basic block - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - p_bb->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert(p_bb->bb_id); - } - - modified = modified || (orig_successor_ids != p_bb->successor_ids); + std::vector::iterator bb_itr; + std::vector::iterator bb_target_itr; + basic_block_t *exit_bb = m_basic_blocks.back(); + + // start from first basic block, which we know is the entry point + bb_itr = m_basic_blocks.begin(); + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + ptx_instruction *pI = (*bb_itr)->ptx_end; + if ((*bb_itr)->is_exit) // reached last basic block, no successors to link + continue; + if (pI->get_opcode() == RETP_OP || pI->get_opcode() == RET_OP || + pI->get_opcode() == EXIT_OP) { + (*bb_itr)->successor_ids.insert(exit_bb->bb_id); + exit_bb->predecessor_ids.insert((*bb_itr)->bb_id); + if (pI->has_pred()) { + printf("GPGPU-Sim PTX: Warning detected predicated return/exit.\n"); + // if predicated, add link to next block + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + if (next_addr < m_instr_mem_size && m_instr_mem[next_addr]) { + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + (*bb_itr)->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } } - } - - return modified; -} -void function_info::do_pdom() -{ - create_basic_blocks(); - connect_basic_blocks(); - bool modified = false; - do { - find_dominators(); - find_idominators(); - modified = connect_break_targets(); - } while (modified == true); - - if ( g_debug_execution>=50 ) { - print_basic_blocks(); - print_basic_block_links(); - print_basic_block_dot(); - } - if ( g_debug_execution>=2 ) { - print_dominators(); - } - find_postdominators(); - find_ipostdominators(); - if ( g_debug_execution>=50 ) { - print_postdominators(); - print_ipostdominators(); - } - printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", m_name.c_str() ); - for ( unsigned ii=0; ii < m_n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions - ptx_instruction *pI = m_instr_mem[ii]; - pI->pre_decode(); - } - printf("GPGPU-Sim PTX: ... done pre-decoding instructions for \'%s\'.\n", m_name.c_str() ); - fflush(stdout); - m_assembled = true; -} -void intersect( std::set &A, const std::set &B ) -{ - // return intersection of A and B in A - for( std::set::iterator a=A.begin(); a!=A.end(); ) { - std::set::iterator a_next = a; - a_next++; - if( B.find(*a) == B.end() ) { - A.erase(*a); - a = a_next; - } else - a++; - } -} - -bool is_equal( const std::set &A, const std::set &B ) -{ - if( A.size() != B.size() ) - return false; - for( std::set::iterator b=B.begin(); b!=B.end(); b++ ) - if( A.find(*b) == A.end() ) - return false; - return true; -} + continue; + } else if (pI->get_opcode() == BRA_OP) { + // find successor and link that basic_block to this one + operand_info &target = pI->dst(); // get operand, e.g. target name + unsigned addr = labels[target.name()]; + ptx_instruction *target_pI = m_instr_mem[addr]; + basic_block_t *target_bb = target_pI->get_bb(); + (*bb_itr)->successor_ids.insert(target_bb->bb_id); + target_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } -void print_set(const std::set &A) + if (!(pI->get_opcode() == BRA_OP && (!pI->has_pred()))) { + // if basic block does not end in an unpredicated branch, + // then next basic block is also successor + // (this is better than testing for .uni) + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + (*bb_itr)->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } else + assert(pI->get_opcode() == BRA_OP); + } +} +bool function_info::connect_break_targets() // connecting break instructions + // with proper targets { - std::set::iterator a; - for (a= A.begin(); a != A.end(); a++) { - printf("%d ", (*a)); - } - printf("\n"); -} - -void function_info::find_dominators( ) -{ - // find dominators using algorithm of Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - printf("GPGPU-Sim PTX: Finding dominators for \'%s\'...\n", m_name.c_str() ); - fflush(stdout); - assert( m_basic_blocks.size() >= 2 ); // must have a distinquished entry block - std::vector::iterator bb_itr = m_basic_blocks.begin(); - (*bb_itr)->dominator_ids.insert((*bb_itr)->bb_id); // the only dominator of the entry block is the entry - //copy all basic blocks to all dominator lists EXCEPT for the entry block - for (++bb_itr;bb_itr != m_basic_blocks.end(); bb_itr++) { - for (unsigned i = 0; i < m_basic_blocks.size(); i++) - (*bb_itr)->dominator_ids.insert(i); - } - bool change = true; - while (change) { - change = false; - for ( int h = 1/*skip entry*/; h < m_basic_blocks.size(); ++h ) { - assert( m_basic_blocks[h]->bb_id == (unsigned)h ); - std::set T; - for (unsigned i=0;i< m_basic_blocks.size();i++) - T.insert(i); - for ( std::set::iterator s = m_basic_blocks[h]->predecessor_ids.begin();s != m_basic_blocks[h]->predecessor_ids.end();s++) - intersect(T, m_basic_blocks[*s]->dominator_ids); - T.insert(h); - if (!is_equal(T, m_basic_blocks[h]->dominator_ids)) { - change = true; - m_basic_blocks[h]->dominator_ids = T; - } + std::vector::iterator bb_itr; + std::vector::iterator bb_target_itr; + bool modified = false; + + // start from first basic block, which we know is the entry point + bb_itr = m_basic_blocks.begin(); + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + basic_block_t *p_bb = *bb_itr; + ptx_instruction *pI = p_bb->ptx_end; + if (p_bb->is_exit) // reached last basic block, no successors to link + continue; + if (pI->get_opcode() == BREAK_OP) { + // backup existing successor_ids for stability check + std::set orig_successor_ids = p_bb->successor_ids; + + // erase the previous linkage with old successors + for (std::set::iterator succ_ids = p_bb->successor_ids.begin(); + succ_ids != p_bb->successor_ids.end(); ++succ_ids) { + basic_block_t *successor_bb = m_basic_blocks[*succ_ids]; + successor_bb->predecessor_ids.erase(p_bb->bb_id); } - } - //clean the basic block of dominators of it has no predecessors -- except for entry block - bb_itr = m_basic_blocks.begin(); - for (++bb_itr;bb_itr != m_basic_blocks.end(); bb_itr++) { - if ((*bb_itr)->predecessor_ids.empty()) - (*bb_itr)->dominator_ids.clear(); - } -} - -void function_info::find_postdominators( ) -{ - // find postdominators using algorithm of Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - printf("GPGPU-Sim PTX: Finding postdominators for \'%s\'...\n", m_name.c_str() ); - fflush(stdout); - assert( m_basic_blocks.size() >= 2 ); // must have a distinquished exit block - std::vector::reverse_iterator bb_itr = m_basic_blocks.rbegin(); - (*bb_itr)->postdominator_ids.insert((*bb_itr)->bb_id); // the only postdominator of the exit block is the exit - for (++bb_itr;bb_itr != m_basic_blocks.rend();bb_itr++) { //copy all basic blocks to all postdominator lists EXCEPT for the exit block - for (unsigned i=0; ipostdominator_ids.insert(i); - } - bool change = true; - while (change) { - change = false; - for ( int h = m_basic_blocks.size()-2/*skip exit*/; h >= 0 ; --h ) { - assert( m_basic_blocks[h]->bb_id == (unsigned)h ); - std::set T; - for (unsigned i=0;i< m_basic_blocks.size();i++) - T.insert(i); - for ( std::set::iterator s = m_basic_blocks[h]->successor_ids.begin();s != m_basic_blocks[h]->successor_ids.end();s++) - intersect(T, m_basic_blocks[*s]->postdominator_ids); - T.insert(h); - if (!is_equal(T,m_basic_blocks[h]->postdominator_ids)) { - change = true; - m_basic_blocks[h]->postdominator_ids = T; - } + p_bb->successor_ids.clear(); + + // find successor and link that basic_block to this one + // successor of a break is set by an preceeding breakaddr instruction + operand_info *target = find_break_target(pI); + unsigned addr = labels[target->name()]; + ptx_instruction *target_pI = m_instr_mem[addr]; + basic_block_t *target_bb = target_pI->get_bb(); + p_bb->successor_ids.insert(target_bb->bb_id); + target_bb->predecessor_ids.insert(p_bb->bb_id); + + if (pI->has_pred()) { + // predicated break - add link to next basic block + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + p_bb->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert(p_bb->bb_id); } - } -} -void function_info::find_ipostdominators( ) -{ - // find immediate postdominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - printf("GPGPU-Sim PTX: Finding immediate postdominators for \'%s\'...\n", m_name.c_str() ); - fflush(stdout); - assert( m_basic_blocks.size() >= 2 ); // must have a distinquished exit block - for (unsigned i=0; iTmp_ids = m_basic_blocks[i]->postdominator_ids; - assert( m_basic_blocks[i]->bb_id == i ); - m_basic_blocks[i]->Tmp_ids.erase(i); - } - for ( int n = m_basic_blocks.size()-2; n >=0;--n) { - // point iterator to basic block before the exit - for( std::set::iterator s=m_basic_blocks[n]->Tmp_ids.begin(); s != m_basic_blocks[n]->Tmp_ids.end(); s++ ) { - int bb_s = *s; - for( std::set::iterator t=m_basic_blocks[n]->Tmp_ids.begin(); t != m_basic_blocks[n]->Tmp_ids.end(); ) { - std::set::iterator t_next = t; t_next++; // might erase thing pointed to be t, invalidating iterator t - if( *s == *t ) { - t = t_next; - continue; - } - int bb_t = *t; - if( m_basic_blocks[bb_s]->postdominator_ids.find(bb_t) != m_basic_blocks[bb_s]->postdominator_ids.end() ) - m_basic_blocks[n]->Tmp_ids.erase(bb_t); - t = t_next; - } + modified = modified || (orig_successor_ids != p_bb->successor_ids); + } + } + + return modified; +} +void function_info::do_pdom() { + create_basic_blocks(); + connect_basic_blocks(); + bool modified = false; + do { + find_dominators(); + find_idominators(); + modified = connect_break_targets(); + } while (modified == true); + + if (g_debug_execution >= 50) { + print_basic_blocks(); + print_basic_block_links(); + print_basic_block_dot(); + } + if (g_debug_execution >= 2) { + print_dominators(); + } + find_postdominators(); + find_ipostdominators(); + if (g_debug_execution >= 50) { + print_postdominators(); + print_ipostdominators(); + } + printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", + m_name.c_str()); + for (unsigned ii = 0; ii < m_n; + ii += m_instr_mem[ii]->inst_size()) { // handle branch instructions + ptx_instruction *pI = m_instr_mem[ii]; + pI->pre_decode(); + } + printf("GPGPU-Sim PTX: ... done pre-decoding instructions for \'%s\'.\n", + m_name.c_str()); + fflush(stdout); + m_assembled = true; +} +void intersect(std::set &A, const std::set &B) { + // return intersection of A and B in A + for (std::set::iterator a = A.begin(); a != A.end();) { + std::set::iterator a_next = a; + a_next++; + if (B.find(*a) == B.end()) { + A.erase(*a); + a = a_next; + } else + a++; + } +} + +bool is_equal(const std::set &A, const std::set &B) { + if (A.size() != B.size()) return false; + for (std::set::iterator b = B.begin(); b != B.end(); b++) + if (A.find(*b) == A.end()) return false; + return true; +} + +void print_set(const std::set &A) { + std::set::iterator a; + for (a = A.begin(); a != A.end(); a++) { + printf("%d ", (*a)); + } + printf("\n"); +} + +void function_info::find_dominators() { + // find dominators using algorithm of Muchnick's Adv. Compiler Design & + // Implemmntation Fig 7.14 + printf("GPGPU-Sim PTX: Finding dominators for \'%s\'...\n", m_name.c_str()); + fflush(stdout); + assert(m_basic_blocks.size() >= 2); // must have a distinquished entry block + std::vector::iterator bb_itr = m_basic_blocks.begin(); + (*bb_itr)->dominator_ids.insert( + (*bb_itr)->bb_id); // the only dominator of the entry block is the entry + // copy all basic blocks to all dominator lists EXCEPT for the entry block + for (++bb_itr; bb_itr != m_basic_blocks.end(); bb_itr++) { + for (unsigned i = 0; i < m_basic_blocks.size(); i++) + (*bb_itr)->dominator_ids.insert(i); + } + bool change = true; + while (change) { + change = false; + for (int h = 1 /*skip entry*/; h < m_basic_blocks.size(); ++h) { + assert(m_basic_blocks[h]->bb_id == (unsigned)h); + std::set T; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) T.insert(i); + for (std::set::iterator s = + m_basic_blocks[h]->predecessor_ids.begin(); + s != m_basic_blocks[h]->predecessor_ids.end(); s++) + intersect(T, m_basic_blocks[*s]->dominator_ids); + T.insert(h); + if (!is_equal(T, m_basic_blocks[h]->dominator_ids)) { + change = true; + m_basic_blocks[h]->dominator_ids = T; } - } - unsigned num_ipdoms=0; - for ( int n = m_basic_blocks.size()-1; n >=0;--n) { - assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); - // if the above assert fails we have an error in either postdominator - // computation, the flow graph does not have a unique exit, or some other error - if( !m_basic_blocks[n]->Tmp_ids.empty() ) { - m_basic_blocks[n]->immediatepostdominator_id = *m_basic_blocks[n]->Tmp_ids.begin(); - num_ipdoms++; + } + } + // clean the basic block of dominators of it has no predecessors -- except for + // entry block + bb_itr = m_basic_blocks.begin(); + for (++bb_itr; bb_itr != m_basic_blocks.end(); bb_itr++) { + if ((*bb_itr)->predecessor_ids.empty()) (*bb_itr)->dominator_ids.clear(); + } +} + +void function_info::find_postdominators() { + // find postdominators using algorithm of Muchnick's Adv. Compiler Design & + // Implemmntation Fig 7.14 + printf("GPGPU-Sim PTX: Finding postdominators for \'%s\'...\n", + m_name.c_str()); + fflush(stdout); + assert(m_basic_blocks.size() >= 2); // must have a distinquished exit block + std::vector::reverse_iterator bb_itr = + m_basic_blocks.rbegin(); + (*bb_itr)->postdominator_ids.insert( + (*bb_itr) + ->bb_id); // the only postdominator of the exit block is the exit + for (++bb_itr; bb_itr != m_basic_blocks.rend(); + bb_itr++) { // copy all basic blocks to all postdominator lists EXCEPT + // for the exit block + for (unsigned i = 0; i < m_basic_blocks.size(); i++) + (*bb_itr)->postdominator_ids.insert(i); + } + bool change = true; + while (change) { + change = false; + for (int h = m_basic_blocks.size() - 2 /*skip exit*/; h >= 0; --h) { + assert(m_basic_blocks[h]->bb_id == (unsigned)h); + std::set T; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) T.insert(i); + for (std::set::iterator s = m_basic_blocks[h]->successor_ids.begin(); + s != m_basic_blocks[h]->successor_ids.end(); s++) + intersect(T, m_basic_blocks[*s]->postdominator_ids); + T.insert(h); + if (!is_equal(T, m_basic_blocks[h]->postdominator_ids)) { + change = true; + m_basic_blocks[h]->postdominator_ids = T; } - } - assert( num_ipdoms == m_basic_blocks.size()-1 ); - // the exit node does not have an immediate post dominator, but everyone else should -} - -void function_info::find_idominators( ) -{ - // find immediate dominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - printf("GPGPU-Sim PTX: Finding immediate dominators for \'%s\'...\n", m_name.c_str() ); - fflush(stdout); - assert( m_basic_blocks.size() >= 2 ); // must have a distinquished entry block - for (unsigned i=0; iTmp_ids = m_basic_blocks[i]->dominator_ids; - assert( m_basic_blocks[i]->bb_id == i ); - m_basic_blocks[i]->Tmp_ids.erase(i); - } - for ( int n = 0; n < m_basic_blocks.size(); ++n) { - // point iterator to basic block before the exit - for( std::set::iterator s=m_basic_blocks[n]->Tmp_ids.begin(); s != m_basic_blocks[n]->Tmp_ids.end(); s++ ) { - int bb_s = *s; - for( std::set::iterator t=m_basic_blocks[n]->Tmp_ids.begin(); t != m_basic_blocks[n]->Tmp_ids.end(); ) { - std::set::iterator t_next = t; t_next++; // might erase thing pointed to be t, invalidating iterator t - if( *s == *t ) { - t = t_next; - continue; - } - int bb_t = *t; - if( m_basic_blocks[bb_s]->dominator_ids.find(bb_t) != m_basic_blocks[bb_s]->dominator_ids.end() ) - m_basic_blocks[n]->Tmp_ids.erase(bb_t); - t = t_next; - } + } + } +} + +void function_info::find_ipostdominators() { + // find immediate postdominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + printf("GPGPU-Sim PTX: Finding immediate postdominators for \'%s\'...\n", + m_name.c_str()); + fflush(stdout); + assert(m_basic_blocks.size() >= 2); // must have a distinquished exit block + for (unsigned i = 0; i < m_basic_blocks.size(); + i++) { // initialize Tmp(n) to all pdoms of n except for n + m_basic_blocks[i]->Tmp_ids = m_basic_blocks[i]->postdominator_ids; + assert(m_basic_blocks[i]->bb_id == i); + m_basic_blocks[i]->Tmp_ids.erase(i); + } + for (int n = m_basic_blocks.size() - 2; n >= 0; --n) { + // point iterator to basic block before the exit + for (std::set::iterator s = m_basic_blocks[n]->Tmp_ids.begin(); + s != m_basic_blocks[n]->Tmp_ids.end(); s++) { + int bb_s = *s; + for (std::set::iterator t = m_basic_blocks[n]->Tmp_ids.begin(); + t != m_basic_blocks[n]->Tmp_ids.end();) { + std::set::iterator t_next = t; + t_next++; // might erase thing pointed to be t, invalidating iterator t + if (*s == *t) { + t = t_next; + continue; + } + int bb_t = *t; + if (m_basic_blocks[bb_s]->postdominator_ids.find(bb_t) != + m_basic_blocks[bb_s]->postdominator_ids.end()) + m_basic_blocks[n]->Tmp_ids.erase(bb_t); + t = t_next; } - } - unsigned num_idoms=0; - unsigned num_nopred = 0; - for ( int n = 0; n < m_basic_blocks.size(); ++n) { - //assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); - // if the above assert fails we have an error in either dominator - // computation, the flow graph does not have a unique entry, or some other error - if( !m_basic_blocks[n]->Tmp_ids.empty() ) { - m_basic_blocks[n]->immediatedominator_id = *m_basic_blocks[n]->Tmp_ids.begin(); - num_idoms++; - } else if (m_basic_blocks[n]->predecessor_ids.empty()) { - num_nopred += 1; + } + } + unsigned num_ipdoms = 0; + for (int n = m_basic_blocks.size() - 1; n >= 0; --n) { + assert(m_basic_blocks[n]->Tmp_ids.size() <= 1); + // if the above assert fails we have an error in either postdominator + // computation, the flow graph does not have a unique exit, or some other + // error + if (!m_basic_blocks[n]->Tmp_ids.empty()) { + m_basic_blocks[n]->immediatepostdominator_id = + *m_basic_blocks[n]->Tmp_ids.begin(); + num_ipdoms++; + } + } + assert(num_ipdoms == m_basic_blocks.size() - 1); + // the exit node does not have an immediate post dominator, but everyone else + // should +} + +void function_info::find_idominators() { + // find immediate dominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + printf("GPGPU-Sim PTX: Finding immediate dominators for \'%s\'...\n", + m_name.c_str()); + fflush(stdout); + assert(m_basic_blocks.size() >= 2); // must have a distinquished entry block + for (unsigned i = 0; i < m_basic_blocks.size(); + i++) { // initialize Tmp(n) to all doms of n except for n + m_basic_blocks[i]->Tmp_ids = m_basic_blocks[i]->dominator_ids; + assert(m_basic_blocks[i]->bb_id == i); + m_basic_blocks[i]->Tmp_ids.erase(i); + } + for (int n = 0; n < m_basic_blocks.size(); ++n) { + // point iterator to basic block before the exit + for (std::set::iterator s = m_basic_blocks[n]->Tmp_ids.begin(); + s != m_basic_blocks[n]->Tmp_ids.end(); s++) { + int bb_s = *s; + for (std::set::iterator t = m_basic_blocks[n]->Tmp_ids.begin(); + t != m_basic_blocks[n]->Tmp_ids.end();) { + std::set::iterator t_next = t; + t_next++; // might erase thing pointed to be t, invalidating iterator t + if (*s == *t) { + t = t_next; + continue; + } + int bb_t = *t; + if (m_basic_blocks[bb_s]->dominator_ids.find(bb_t) != + m_basic_blocks[bb_s]->dominator_ids.end()) + m_basic_blocks[n]->Tmp_ids.erase(bb_t); + t = t_next; } - } - assert( num_idoms == m_basic_blocks.size()-num_nopred ); - // the entry node does not have an immediate dominator, but everyone else should -} - -void function_info::print_dominators() -{ - printf("Printing dominators for function \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - for( std::set::iterator j=m_basic_blocks[i]->dominator_ids.begin(); j!=m_basic_blocks[i]->dominator_ids.end(); j++) - printf(" %d", *j ); - printf("\n"); - } -} - -void function_info::print_postdominators() -{ - printf("Printing postdominators for function \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - for( std::set::iterator j=m_basic_blocks[i]->postdominator_ids.begin(); j!=m_basic_blocks[i]->postdominator_ids.end(); j++) - printf(" %d", *j ); - printf("\n"); - } -} - -void function_info::print_ipostdominators() -{ - printf("Printing immediate postdominators for function \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - printf("%d\n", m_basic_blocks[i]->immediatepostdominator_id); - } -} - -void function_info::print_idominators() -{ - printf("Printing immediate dominators for function \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - printf("%d\n", m_basic_blocks[i]->immediatedominator_id); - } -} - -unsigned function_info::get_num_reconvergence_pairs() -{ - if (!num_reconvergence_pairs) { - if( m_basic_blocks.size() == 0 ) - return 0; - for (unsigned i=0; i< (m_basic_blocks.size()-1); i++) { //last basic block containing exit obviously won't have a pair - if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { - num_reconvergence_pairs++; - } + } + } + unsigned num_idoms = 0; + unsigned num_nopred = 0; + for (int n = 0; n < m_basic_blocks.size(); ++n) { + // assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); + // if the above assert fails we have an error in either dominator + // computation, the flow graph does not have a unique entry, or some other + // error + if (!m_basic_blocks[n]->Tmp_ids.empty()) { + m_basic_blocks[n]->immediatedominator_id = + *m_basic_blocks[n]->Tmp_ids.begin(); + num_idoms++; + } else if (m_basic_blocks[n]->predecessor_ids.empty()) { + num_nopred += 1; + } + } + assert(num_idoms == m_basic_blocks.size() - num_nopred); + // the entry node does not have an immediate dominator, but everyone else + // should +} + +void function_info::print_dominators() { + printf("Printing dominators for function \'%s\':\n", m_name.c_str()); + std::vector::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + for (std::set::iterator j = m_basic_blocks[i]->dominator_ids.begin(); + j != m_basic_blocks[i]->dominator_ids.end(); j++) + printf(" %d", *j); + printf("\n"); + } +} + +void function_info::print_postdominators() { + printf("Printing postdominators for function \'%s\':\n", m_name.c_str()); + std::vector::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + for (std::set::iterator j = + m_basic_blocks[i]->postdominator_ids.begin(); + j != m_basic_blocks[i]->postdominator_ids.end(); j++) + printf(" %d", *j); + printf("\n"); + } +} + +void function_info::print_ipostdominators() { + printf("Printing immediate postdominators for function \'%s\':\n", + m_name.c_str()); + std::vector::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + printf("%d\n", m_basic_blocks[i]->immediatepostdominator_id); + } +} + +void function_info::print_idominators() { + printf("Printing immediate dominators for function \'%s\':\n", + m_name.c_str()); + std::vector::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + printf("%d\n", m_basic_blocks[i]->immediatedominator_id); + } +} + +unsigned function_info::get_num_reconvergence_pairs() { + if (!num_reconvergence_pairs) { + if (m_basic_blocks.size() == 0) return 0; + for (unsigned i = 0; i < (m_basic_blocks.size() - 1); + i++) { // last basic block containing exit obviously won't have a pair + if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { + num_reconvergence_pairs++; } - } - return num_reconvergence_pairs; + } + } + return num_reconvergence_pairs; } -void function_info::get_reconvergence_pairs(gpgpu_recon_t* recon_points) -{ - unsigned idx=0; //array index - if( m_basic_blocks.size() == 0 ) - return; - for (unsigned i=0; i< (m_basic_blocks.size()-1); i++) { //last basic block containing exit obviously won't have a pair +void function_info::get_reconvergence_pairs(gpgpu_recon_t *recon_points) { + unsigned idx = 0; // array index + if (m_basic_blocks.size() == 0) return; + for (unsigned i = 0; i < (m_basic_blocks.size() - 1); + i++) { // last basic block containing exit obviously won't have a pair #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("i=%d\n", i); fflush(stdout); + printf("i=%d\n", i); + fflush(stdout); #endif - if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { + if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("\tbranch!\n"); - printf("\tbb_id=%d; ipdom=%d\n", m_basic_blocks[i]->bb_id, m_basic_blocks[i]->immediatepostdominator_id); - printf("\tm_instr_mem index=%d\n", m_basic_blocks[i]->ptx_end->get_m_instr_mem_index()); - fflush(stdout); + printf("\tbranch!\n"); + printf("\tbb_id=%d; ipdom=%d\n", m_basic_blocks[i]->bb_id, + m_basic_blocks[i]->immediatepostdominator_id); + printf("\tm_instr_mem index=%d\n", + m_basic_blocks[i]->ptx_end->get_m_instr_mem_index()); + fflush(stdout); #endif - recon_points[idx].source_pc = m_basic_blocks[i]->ptx_end->get_PC(); - recon_points[idx].source_inst = m_basic_blocks[i]->ptx_end; + recon_points[idx].source_pc = m_basic_blocks[i]->ptx_end->get_PC(); + recon_points[idx].source_inst = m_basic_blocks[i]->ptx_end; #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("\trecon_points[idx].source_pc=%d\n", recon_points[idx].source_pc); + printf("\trecon_points[idx].source_pc=%d\n", recon_points[idx].source_pc); #endif - if( m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin ) { - recon_points[idx].target_pc = m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin->get_PC(); - recon_points[idx].target_inst = m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin; - } else { - // reconverge after function return - recon_points[idx].target_pc = -2; - recon_points[idx].target_inst = NULL; - } + if (m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] + ->ptx_begin) { + recon_points[idx].target_pc = + m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] + ->ptx_begin->get_PC(); + recon_points[idx].target_inst = + m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] + ->ptx_begin; + } else { + // reconverge after function return + recon_points[idx].target_pc = -2; + recon_points[idx].target_inst = NULL; + } #ifdef DEBUG_GET_RECONVERG_PAIRS - m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin->print_insn(); - printf("\trecon_points[idx].target_pc=%d\n", recon_points[idx].target_pc); fflush(stdout); + m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] + ->ptx_begin->print_insn(); + printf("\trecon_points[idx].target_pc=%d\n", recon_points[idx].target_pc); + fflush(stdout); #endif - idx++; - } - } + idx++; + } + } } // interface with graphviz (print the graph in DOT language) for plotting -void function_info::print_basic_block_dot() -{ - printf("Basic Block in DOT\n"); - printf("digraph %s {\n", m_name.c_str()); - std::vector::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - printf("\t"); - std::set::iterator s; - for (s = (*bb_itr)->successor_ids.begin();s != (*bb_itr)->successor_ids.end();s++) { - unsigned succ_bb = *s; - printf("%d -> %d; ", (*bb_itr)->bb_id, succ_bb ); - } - printf("\n"); - } - printf("}\n"); -} - -unsigned ptx_kernel_shmem_size( void *kernel_impl ) -{ - function_info *f = (function_info*)kernel_impl; - const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); - return kernel_info->smem; -} - -unsigned ptx_kernel_nregs( void *kernel_impl ) -{ - function_info *f = (function_info*)kernel_impl; - const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); - return kernel_info->regs; -} - -unsigned type_info_key::type_decode( size_t &size, int &basic_type ) const -{ - int type = scalar_type(); - return type_decode(type,size,basic_type); -} - -unsigned type_info_key::type_decode( int type, size_t &size, int &basic_type ) -{ - switch ( type ) { - case S8_TYPE: size=8; basic_type=1; return 0; - case S16_TYPE: size=16; basic_type=1; return 1; - case S32_TYPE: size=32; basic_type=1; return 2; - case S64_TYPE: size=64; basic_type=1; return 3; - case U8_TYPE: size=8; basic_type=0; return 4; - case U16_TYPE: size=16; basic_type=0; return 5; - case U32_TYPE: size=32; basic_type=0; return 6; - case U64_TYPE: size=64; basic_type=0; return 7; - case F16_TYPE: size=16; basic_type=-1; return 8; - case F32_TYPE: size=32; basic_type=-1; return 9; - case F64_TYPE: size=64; basic_type=-1; return 10; - case FF64_TYPE: size=64; basic_type=-1; return 10; - case PRED_TYPE: size=1; basic_type=2; return 11; - case B8_TYPE: size=8; basic_type=0; return 12; - case B16_TYPE: size=16; basic_type=0; return 13; - case B32_TYPE: size=32; basic_type=0; return 14; - case B64_TYPE: size=64; basic_type=0; return 15; - case BB64_TYPE: size=64; basic_type=0; return 15; - case BB128_TYPE: size=128; basic_type=0; return 16; - case TEXREF_TYPE: case SAMPLERREF_TYPE: case SURFREF_TYPE: - size=32; basic_type=3; return 16; - default: - printf("ERROR ** type_decode() does not know about \"%s\"\n", decode_token(type) ); - assert(0); +void function_info::print_basic_block_dot() { + printf("Basic Block in DOT\n"); + printf("digraph %s {\n", m_name.c_str()); + std::vector::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + printf("\t"); + std::set::iterator s; + for (s = (*bb_itr)->successor_ids.begin(); + s != (*bb_itr)->successor_ids.end(); s++) { + unsigned succ_bb = *s; + printf("%d -> %d; ", (*bb_itr)->bb_id, succ_bb); + } + printf("\n"); + } + printf("}\n"); +} + +unsigned ptx_kernel_shmem_size(void *kernel_impl) { + function_info *f = (function_info *)kernel_impl; + const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); + return kernel_info->smem; +} + +unsigned ptx_kernel_nregs(void *kernel_impl) { + function_info *f = (function_info *)kernel_impl; + const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); + return kernel_info->regs; +} + +unsigned type_info_key::type_decode(size_t &size, int &basic_type) const { + int type = scalar_type(); + return type_decode(type, size, basic_type); +} + +unsigned type_info_key::type_decode(int type, size_t &size, int &basic_type) { + switch (type) { + case S8_TYPE: + size = 8; + basic_type = 1; + return 0; + case S16_TYPE: + size = 16; + basic_type = 1; + return 1; + case S32_TYPE: + size = 32; + basic_type = 1; + return 2; + case S64_TYPE: + size = 64; + basic_type = 1; + return 3; + case U8_TYPE: + size = 8; + basic_type = 0; + return 4; + case U16_TYPE: + size = 16; + basic_type = 0; + return 5; + case U32_TYPE: + size = 32; + basic_type = 0; + return 6; + case U64_TYPE: + size = 64; + basic_type = 0; + return 7; + case F16_TYPE: + size = 16; + basic_type = -1; + return 8; + case F32_TYPE: + size = 32; + basic_type = -1; + return 9; + case F64_TYPE: + size = 64; + basic_type = -1; + return 10; + case FF64_TYPE: + size = 64; + basic_type = -1; + return 10; + case PRED_TYPE: + size = 1; + basic_type = 2; + return 11; + case B8_TYPE: + size = 8; + basic_type = 0; + return 12; + case B16_TYPE: + size = 16; + basic_type = 0; + return 13; + case B32_TYPE: + size = 32; + basic_type = 0; + return 14; + case B64_TYPE: + size = 64; + basic_type = 0; + return 15; + case BB64_TYPE: + size = 64; + basic_type = 0; + return 15; + case BB128_TYPE: + size = 128; + basic_type = 0; + return 16; + case TEXREF_TYPE: + case SAMPLERREF_TYPE: + case SURFREF_TYPE: + size = 32; + basic_type = 3; + return 16; + default: + printf("ERROR ** type_decode() does not know about \"%s\"\n", + decode_token(type)); + assert(0); return 0xDEADBEEF; - } -} - -arg_buffer_t copy_arg_to_buffer(ptx_thread_info * thread, operand_info actual_param_op, const symbol * formal_param) -{ - if( actual_param_op.is_reg() ) { - ptx_reg_t value = thread->get_reg(actual_param_op.get_symbol()); - return arg_buffer_t(formal_param,actual_param_op,value); - } else if ( actual_param_op.is_param_local() ) { - unsigned size=formal_param->get_size_in_bytes(); - addr_t frame_offset = actual_param_op.get_symbol()->get_address(); - addr_t from_addr = thread->get_local_mem_stack_pointer() + frame_offset; - char buffer[1024]; - assert(size<1024); - thread->m_local_mem->read(from_addr,size,buffer); - return arg_buffer_t(formal_param,actual_param_op,buffer,size); - } else { - printf("GPGPU-Sim PTX: ERROR ** need to add support for this operand type in call/return\n"); - abort(); - } -} - -void copy_args_into_buffer_list( const ptx_instruction * pI, - ptx_thread_info * thread, - const function_info * target_func, - arg_buffer_list_t &arg_values ) -{ - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); - const symbol *formal_param = target_func->get_arg(arg); - arg_values.push_back( copy_arg_to_buffer(thread, actual_param_op, formal_param) ); - } -} - -void copy_buffer_to_frame(ptx_thread_info * thread, const arg_buffer_t &a) -{ - if( a.is_reg() ) { - ptx_reg_t value = a.get_reg(); - operand_info dst_reg = operand_info(a.get_dst(), thread->get_gpu()->gpgpu_ctx); - thread->set_reg(dst_reg.get_symbol(),value); - } else { - const void *buffer = a.get_param_buffer(); - size_t size = a.get_param_buffer_size(); - const symbol *dst = a.get_dst(); - addr_t frame_offset = dst->get_address(); - addr_t to_addr = thread->get_local_mem_stack_pointer() + frame_offset; - thread->m_local_mem->write(to_addr,size,buffer,NULL,NULL); - } -} - -void copy_buffer_list_into_frame(ptx_thread_info * thread, arg_buffer_list_t &arg_values) -{ - arg_buffer_list_t::iterator a; - for( a=arg_values.begin(); a != arg_values.end(); a++ ) { - copy_buffer_to_frame(thread, *a); - } -} - - - -static std::list check_operands( int opcode, - const std::list &scalar_type, - const std::list &operands, - gpgpu_context* ctx) -{ - static int g_warn_literal_operands_two_type_inst; - if( (opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || (opcode == TEX_OP) || (opcode==MMA_OP) || (opcode == DP4A_OP)) { - // just make sure these do not have have const operands... - if( !g_warn_literal_operands_two_type_inst ) { - std::list::const_iterator o; - for( o = operands.begin(); o != operands.end(); o++ ) { - const operand_info &op = *o; - if( op.is_literal() ) { - printf("GPGPU-Sim PTX: PTX uses two scalar type intruction with literal operand.\n"); - g_warn_literal_operands_two_type_inst = 1; - } - } + } +} + +arg_buffer_t copy_arg_to_buffer(ptx_thread_info *thread, + operand_info actual_param_op, + const symbol *formal_param) { + if (actual_param_op.is_reg()) { + ptx_reg_t value = thread->get_reg(actual_param_op.get_symbol()); + return arg_buffer_t(formal_param, actual_param_op, value); + } else if (actual_param_op.is_param_local()) { + unsigned size = formal_param->get_size_in_bytes(); + addr_t frame_offset = actual_param_op.get_symbol()->get_address(); + addr_t from_addr = thread->get_local_mem_stack_pointer() + frame_offset; + char buffer[1024]; + assert(size < 1024); + thread->m_local_mem->read(from_addr, size, buffer); + return arg_buffer_t(formal_param, actual_param_op, buffer, size); + } else { + printf( + "GPGPU-Sim PTX: ERROR ** need to add support for this operand type in " + "call/return\n"); + abort(); + } +} + +void copy_args_into_buffer_list(const ptx_instruction *pI, + ptx_thread_info *thread, + const function_info *target_func, + arg_buffer_list_t &arg_values) { + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); + const symbol *formal_param = target_func->get_arg(arg); + arg_values.push_back( + copy_arg_to_buffer(thread, actual_param_op, formal_param)); + } +} + +void copy_buffer_to_frame(ptx_thread_info *thread, const arg_buffer_t &a) { + if (a.is_reg()) { + ptx_reg_t value = a.get_reg(); + operand_info dst_reg = + operand_info(a.get_dst(), thread->get_gpu()->gpgpu_ctx); + thread->set_reg(dst_reg.get_symbol(), value); + } else { + const void *buffer = a.get_param_buffer(); + size_t size = a.get_param_buffer_size(); + const symbol *dst = a.get_dst(); + addr_t frame_offset = dst->get_address(); + addr_t to_addr = thread->get_local_mem_stack_pointer() + frame_offset; + thread->m_local_mem->write(to_addr, size, buffer, NULL, NULL); + } +} + +void copy_buffer_list_into_frame(ptx_thread_info *thread, + arg_buffer_list_t &arg_values) { + arg_buffer_list_t::iterator a; + for (a = arg_values.begin(); a != arg_values.end(); a++) { + copy_buffer_to_frame(thread, *a); + } +} + +static std::list check_operands( + int opcode, const std::list &scalar_type, + const std::list &operands, gpgpu_context *ctx) { + static int g_warn_literal_operands_two_type_inst; + if ((opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || + (opcode == TEX_OP) || (opcode == MMA_OP) || (opcode == DP4A_OP)) { + // just make sure these do not have have const operands... + if (!g_warn_literal_operands_two_type_inst) { + std::list::const_iterator o; + for (o = operands.begin(); o != operands.end(); o++) { + const operand_info &op = *o; + if (op.is_literal()) { + printf( + "GPGPU-Sim PTX: PTX uses two scalar type intruction with literal " + "operand.\n"); + g_warn_literal_operands_two_type_inst = 1; } - } else { - assert( scalar_type.size() < 2 ); - if( scalar_type.size() == 1 ) { - std::list result; - int inst_type = scalar_type.front(); - std::list::const_iterator o; - for( o = operands.begin(); o != operands.end(); o++ ) { - const operand_info &op = *o; - if( op.is_literal() ) { - if( (op.get_type() == double_op_t) && (inst_type == F32_TYPE) ) { - ptx_reg_t v = op.get_literal_value(); - float u = (float)v.f64; - operand_info n(u, ctx); - result.push_back(n); - } else { - result.push_back(op); - } - } else { - result.push_back(op); - } - } - return result; - } + } } - return operands; -} - - -ptx_instruction::ptx_instruction( int opcode, - const symbol *pred, - int neg_pred, - int pred_mod, - symbol *label, - const std::list &operands, - const operand_info &return_var, - const std::list &options, - const std::list &wmma_options, - const std::list &scalar_type, - memory_space_t space_spec, - const char *file, - unsigned line, - const char *source, - const core_config *config, - gpgpu_context* ctx ) : warp_inst_t(config), m_return_var(ctx) -{ - gpgpu_ctx = ctx; - m_uid = ++(ctx->g_num_ptx_inst_uid); - m_PC = 0; - m_opcode = opcode; - m_pred = pred; - m_neg_pred = neg_pred; - m_pred_mod = pred_mod; - m_label = label; - const std::list checked_operands = check_operands(opcode,scalar_type,operands, ctx); - m_operands.insert(m_operands.begin(), checked_operands.begin(), checked_operands.end() ); - m_return_var = return_var; - m_options = options; - m_wmma_options = wmma_options; - m_wide = false; - m_hi = false; - m_lo = false; - m_uni = false; - m_exit = false; - m_abs = false; - m_neg = false; - m_to_option = false; - m_cache_option = 0; - m_rounding_mode = RN_OPTION; - m_compare_op = -1; - m_saturation_mode = 0; - m_geom_spec = 0; - m_vector_spec = 0; - m_atomic_spec = 0; - m_membar_level = 0; - m_inst_size = 8; // bytes - int rr=0; - std::list::const_iterator i; - unsigned n=1; - for ( i=wmma_options.begin(); i!= wmma_options.end(); i++, n++ ) { - int last_ptx_inst_option = *i; - switch ( last_ptx_inst_option ) { - case SYNC_OPTION: - case LOAD_A: - case LOAD_B: - case LOAD_C: - case STORE_D: - case MMA: - m_wmma_type=last_ptx_inst_option; - break; - case ROW: - case COL: - m_wmma_layout[rr++]=last_ptx_inst_option; - break; - case M16N16K16: - case M32N8K16: - case M8N32K16: - break; - default: - assert(0); - break; - } - } - rr=0; - n=1; - for ( i=options.begin(); i!= options.end(); i++, n++ ) { - int last_ptx_inst_option = *i; - switch ( last_ptx_inst_option ) { + } else { + assert(scalar_type.size() < 2); + if (scalar_type.size() == 1) { + std::list result; + int inst_type = scalar_type.front(); + std::list::const_iterator o; + for (o = operands.begin(); o != operands.end(); o++) { + const operand_info &op = *o; + if (op.is_literal()) { + if ((op.get_type() == double_op_t) && (inst_type == F32_TYPE)) { + ptx_reg_t v = op.get_literal_value(); + float u = (float)v.f64; + operand_info n(u, ctx); + result.push_back(n); + } else { + result.push_back(op); + } + } else { + result.push_back(op); + } + } + return result; + } + } + return operands; +} + +ptx_instruction::ptx_instruction( + int opcode, const symbol *pred, int neg_pred, int pred_mod, symbol *label, + const std::list &operands, const operand_info &return_var, + const std::list &options, const std::list &wmma_options, + const std::list &scalar_type, memory_space_t space_spec, + const char *file, unsigned line, const char *source, + const core_config *config, gpgpu_context *ctx) + : warp_inst_t(config), m_return_var(ctx) { + gpgpu_ctx = ctx; + m_uid = ++(ctx->g_num_ptx_inst_uid); + m_PC = 0; + m_opcode = opcode; + m_pred = pred; + m_neg_pred = neg_pred; + m_pred_mod = pred_mod; + m_label = label; + const std::list checked_operands = + check_operands(opcode, scalar_type, operands, ctx); + m_operands.insert(m_operands.begin(), checked_operands.begin(), + checked_operands.end()); + m_return_var = return_var; + m_options = options; + m_wmma_options = wmma_options; + m_wide = false; + m_hi = false; + m_lo = false; + m_uni = false; + m_exit = false; + m_abs = false; + m_neg = false; + m_to_option = false; + m_cache_option = 0; + m_rounding_mode = RN_OPTION; + m_compare_op = -1; + m_saturation_mode = 0; + m_geom_spec = 0; + m_vector_spec = 0; + m_atomic_spec = 0; + m_membar_level = 0; + m_inst_size = 8; // bytes + int rr = 0; + std::list::const_iterator i; + unsigned n = 1; + for (i = wmma_options.begin(); i != wmma_options.end(); i++, n++) { + int last_ptx_inst_option = *i; + switch (last_ptx_inst_option) { + case SYNC_OPTION: + case LOAD_A: + case LOAD_B: + case LOAD_C: + case STORE_D: + case MMA: + m_wmma_type = last_ptx_inst_option; + break; + case ROW: + case COL: + m_wmma_layout[rr++] = last_ptx_inst_option; + break; + case M16N16K16: + case M32N8K16: + case M8N32K16: + break; + default: + assert(0); + break; + } + } + rr = 0; + n = 1; + for (i = options.begin(); i != options.end(); i++, n++) { + int last_ptx_inst_option = *i; + switch (last_ptx_inst_option) { case SYNC_OPTION: case ARRIVE_OPTION: case RED_OPTION: - m_barrier_op = last_ptx_inst_option; - break; + m_barrier_op = last_ptx_inst_option; + break; case EQU_OPTION: case NEU_OPTION: case LTU_OPTION: @@ -1186,16 +1280,16 @@ ptx_instruction::ptx_instruction( int opcode, case GE_OPTION: case LS_OPTION: case HS_OPTION: - m_compare_op = last_ptx_inst_option; - break; + m_compare_op = last_ptx_inst_option; + break; case NUM_OPTION: case NAN_OPTION: - m_compare_op = last_ptx_inst_option; + m_compare_op = last_ptx_inst_option; // assert(0); // finish this - break; + break; case SAT_OPTION: - m_saturation_mode = 1; - break; + m_saturation_mode = 1; + break; case RNI_OPTION: case RZI_OPTION: case RMI_OPTION: @@ -1204,38 +1298,39 @@ ptx_instruction::ptx_instruction( int opcode, case RZ_OPTION: case RM_OPTION: case RP_OPTION: - m_rounding_mode = last_ptx_inst_option; - break; + m_rounding_mode = last_ptx_inst_option; + break; case HI_OPTION: - m_compare_op = last_ptx_inst_option; - m_hi = true; - assert( !m_lo ); - assert( !m_wide ); - break; + m_compare_op = last_ptx_inst_option; + m_hi = true; + assert(!m_lo); + assert(!m_wide); + break; case LO_OPTION: - m_compare_op = last_ptx_inst_option; - m_lo = true; - assert( !m_hi ); - assert( !m_wide ); - break; + m_compare_op = last_ptx_inst_option; + m_lo = true; + assert(!m_hi); + assert(!m_wide); + break; case WIDE_OPTION: - m_wide = true; - assert( !m_lo ); - assert( !m_hi ); - break; + m_wide = true; + assert(!m_lo); + assert(!m_hi); + break; case UNI_OPTION: - m_uni = true; // don't care... < now we DO care when constructing flowgraph> - break; + m_uni = true; // don't care... < now we DO care when constructing + // flowgraph> + break; case GEOM_MODIFIER_1D: case GEOM_MODIFIER_2D: case GEOM_MODIFIER_3D: - m_geom_spec = last_ptx_inst_option; - break; + m_geom_spec = last_ptx_inst_option; + break; case V2_TYPE: case V3_TYPE: case V4_TYPE: - m_vector_spec = last_ptx_inst_option; - break; + m_vector_spec = last_ptx_inst_option; + break; case ATOMIC_AND: case ATOMIC_OR: case ATOMIC_XOR: @@ -1246,223 +1341,225 @@ ptx_instruction::ptx_instruction( int opcode, case ATOMIC_DEC: case ATOMIC_MIN: case ATOMIC_MAX: - m_atomic_spec = last_ptx_inst_option; - break; + m_atomic_spec = last_ptx_inst_option; + break; case APPROX_OPTION: - break; + break; case FULL_OPTION: - break; + break; case ANY_OPTION: - m_vote_mode = vote_any; - break; + m_vote_mode = vote_any; + break; case ALL_OPTION: - m_vote_mode = vote_all; - break; + m_vote_mode = vote_all; + break; case BALLOT_OPTION: - m_vote_mode = vote_ballot; - break; + m_vote_mode = vote_ballot; + break; case GLOBAL_OPTION: - m_membar_level = GLOBAL_OPTION; - break; + m_membar_level = GLOBAL_OPTION; + break; case CTA_OPTION: - m_membar_level = CTA_OPTION; - break; + m_membar_level = CTA_OPTION; + break; case SYS_OPTION: - m_membar_level = SYS_OPTION; - break; + m_membar_level = SYS_OPTION; + break; case FTZ_OPTION: - break; + break; case EXIT_OPTION: - m_exit = true; - break; + m_exit = true; + break; case ABS_OPTION: - m_abs = true; - break; + m_abs = true; + break; case NEG_OPTION: - m_neg = true; - break; + m_neg = true; + break; case TO_OPTION: - m_to_option = true; - break; - case CA_OPTION: case CG_OPTION: case CS_OPTION: case LU_OPTION: case CV_OPTION: - m_cache_option = last_ptx_inst_option; - break; + m_to_option = true; + break; + case CA_OPTION: + case CG_OPTION: + case CS_OPTION: + case LU_OPTION: + case CV_OPTION: + m_cache_option = last_ptx_inst_option; + break; case HALF_OPTION: - m_inst_size = 4; // bytes - break; + m_inst_size = 4; // bytes + break; case EXTP_OPTION: - break; + break; case NC_OPTION: - m_cache_option = last_ptx_inst_option; - break; + m_cache_option = last_ptx_inst_option; + break; case UP_OPTION: case DOWN_OPTION: case BFLY_OPTION: case IDX_OPTION: - m_shfl_op = last_ptx_inst_option; - break; + m_shfl_op = last_ptx_inst_option; + break; case PRMT_F4E_MODE: case PRMT_B4E_MODE: case PRMT_RC8_MODE: case PRMT_ECL_MODE: case PRMT_ECR_MODE: case PRMT_RC16_MODE: - m_prmt_op = last_ptx_inst_option; - break; + m_prmt_op = last_ptx_inst_option; + break; default: - assert(0); - break; - } - } - m_scalar_type = scalar_type; - m_space_spec = space_spec; - if( ( opcode == ST_OP || opcode == LD_OP || opcode == LDU_OP ) && (space_spec == undefined_space) ) { - m_space_spec = generic_space; - } - for( std::vector::const_iterator i=m_operands.begin(); i!=m_operands.end(); ++i) { - const operand_info &op = *i; - if( op.get_addr_space() != undefined_space ) - m_space_spec = op.get_addr_space(); // TODO: can have more than one memory space for ptxplus (g8x) inst - } - if( opcode == TEX_OP ) - m_space_spec = tex_space; - - m_source_file = file?file:""; - m_source_line = line; - m_source = source; - // Trim tabs - m_source.erase( std::remove( m_source.begin(), m_source.end(), '\t' ), m_source.end() ); - - if (opcode == CALL_OP) { - const operand_info &target = func_addr(); - assert( target.is_function_address() ); - const symbol *func_addr = target.get_symbol(); - const function_info *target_func = func_addr->get_pc(); - std::string fname = target_func->get_name(); - - if (fname =="vprintf"){ - m_is_printf = true; - } - if(fname == "cudaStreamCreateWithFlags") - m_is_cdp = 1; - if(fname == "cudaGetParameterBufferV2") - m_is_cdp = 2; - if(fname == "cudaLaunchDeviceV2") - m_is_cdp = 4; - - } -} - -void ptx_instruction::print_insn() const -{ - print_insn(stdout); - fflush(stdout); -} - -void ptx_instruction::print_insn( FILE *fp ) const -{ - fprintf( fp, "%s", to_string().c_str() ); -} - -std::string ptx_instruction::to_string() const -{ - char buf[ STR_SIZE ]; - unsigned used_bytes = 0; - if( !is_label() ) { - used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, " PC=0x%03x ", m_PC ); - } else { - used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, " " ); - } - used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, - "(%s:%d) %s", - m_source_file.c_str(), m_source_line, - m_source.c_str() ); - return std::string( buf ); -} -operand_info ptx_instruction::get_pred() const -{ - return operand_info( m_pred, gpgpu_ctx); -} - - -function_info::function_info(int entry_point, gpgpu_context* ctx ) -{ - gpgpu_ctx = ctx; - m_uid = (gpgpu_ctx->function_info_sm_next_uid)++; - m_entry_point = (entry_point==1)?true:false; - m_extern = (entry_point==2)?true:false; - num_reconvergence_pairs = 0; - m_symtab = NULL; - m_assembled = false; - m_return_var_sym = NULL; - m_kernel_info.cmem = 0; - m_kernel_info.lmem = 0; - m_kernel_info.regs = 0; - m_kernel_info.smem = 0; - m_local_mem_framesize = 0; - m_args_aligned_size = -1; - pdom_done = false; //initialize it to false -} - -unsigned function_info::print_insn( unsigned pc, FILE * fp ) const -{ - unsigned inst_size=1; // return offset to next instruction or 1 if unknown - unsigned index = pc - m_start_PC; - char command[1024]; - char buffer[1024]; - memset(command, 0, 1024); - memset(buffer, 0, 1024); - snprintf(command,1024,"c++filt -p %s",m_name.c_str()); - FILE *p = popen(command,"r"); - buffer[0]=0; - assert(fgets(buffer, 1023, p) != NULL); - // Remove trailing "\n" in buffer - char *c; - if ((c=strchr(buffer, '\n')) != NULL) *c = '\0'; - fprintf(fp,"%s",buffer); - if ( index >= m_instr_mem_size ) { - fprintf(fp, "", m_start_PC + m_instr_mem_size - 1 ); - } else { - if ( m_instr_mem[index] != NULL ) { - m_instr_mem[index]->print_insn(fp); - inst_size = m_instr_mem[index]->isize; - } else - fprintf(fp, "", pc ); - } - pclose(p); - return inst_size; -} - -std::string function_info::get_insn_str( unsigned pc ) const -{ - unsigned index = pc - m_start_PC; - if ( index >= m_instr_mem_size ) { + assert(0); + break; + } + } + m_scalar_type = scalar_type; + m_space_spec = space_spec; + if ((opcode == ST_OP || opcode == LD_OP || opcode == LDU_OP) && + (space_spec == undefined_space)) { + m_space_spec = generic_space; + } + for (std::vector::const_iterator i = m_operands.begin(); + i != m_operands.end(); ++i) { + const operand_info &op = *i; + if (op.get_addr_space() != undefined_space) + m_space_spec = op.get_addr_space(); // TODO: can have more than one + // memory space for ptxplus (g8x) + // inst + } + if (opcode == TEX_OP) m_space_spec = tex_space; + + m_source_file = file ? file : ""; + m_source_line = line; + m_source = source; + // Trim tabs + m_source.erase(std::remove(m_source.begin(), m_source.end(), '\t'), + m_source.end()); + + if (opcode == CALL_OP) { + const operand_info &target = func_addr(); + assert(target.is_function_address()); + const symbol *func_addr = target.get_symbol(); + const function_info *target_func = func_addr->get_pc(); + std::string fname = target_func->get_name(); + + if (fname == "vprintf") { + m_is_printf = true; + } + if (fname == "cudaStreamCreateWithFlags") m_is_cdp = 1; + if (fname == "cudaGetParameterBufferV2") m_is_cdp = 2; + if (fname == "cudaLaunchDeviceV2") m_is_cdp = 4; + } +} + +void ptx_instruction::print_insn() const { + print_insn(stdout); + fflush(stdout); +} + +void ptx_instruction::print_insn(FILE *fp) const { + fprintf(fp, "%s", to_string().c_str()); +} + +std::string ptx_instruction::to_string() const { + char buf[STR_SIZE]; + unsigned used_bytes = 0; + if (!is_label()) { + used_bytes += + snprintf(buf + used_bytes, STR_SIZE - used_bytes, " PC=0x%03x ", m_PC); + } else { + used_bytes += + snprintf(buf + used_bytes, STR_SIZE - used_bytes, " "); + } + used_bytes += + snprintf(buf + used_bytes, STR_SIZE - used_bytes, "(%s:%d) %s", + m_source_file.c_str(), m_source_line, m_source.c_str()); + return std::string(buf); +} +operand_info ptx_instruction::get_pred() const { + return operand_info(m_pred, gpgpu_ctx); +} + +function_info::function_info(int entry_point, gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_uid = (gpgpu_ctx->function_info_sm_next_uid)++; + m_entry_point = (entry_point == 1) ? true : false; + m_extern = (entry_point == 2) ? true : false; + num_reconvergence_pairs = 0; + m_symtab = NULL; + m_assembled = false; + m_return_var_sym = NULL; + m_kernel_info.cmem = 0; + m_kernel_info.lmem = 0; + m_kernel_info.regs = 0; + m_kernel_info.smem = 0; + m_local_mem_framesize = 0; + m_args_aligned_size = -1; + pdom_done = false; // initialize it to false +} + +unsigned function_info::print_insn(unsigned pc, FILE *fp) const { + unsigned inst_size = 1; // return offset to next instruction or 1 if unknown + unsigned index = pc - m_start_PC; + char command[1024]; + char buffer[1024]; + memset(command, 0, 1024); + memset(buffer, 0, 1024); + snprintf(command, 1024, "c++filt -p %s", m_name.c_str()); + FILE *p = popen(command, "r"); + buffer[0] = 0; + assert(fgets(buffer, 1023, p) != NULL); + // Remove trailing "\n" in buffer + char *c; + if ((c = strchr(buffer, '\n')) != NULL) *c = '\0'; + fprintf(fp, "%s", buffer); + if (index >= m_instr_mem_size) { + fprintf(fp, "", + m_start_PC + m_instr_mem_size - 1); + } else { + if (m_instr_mem[index] != NULL) { + m_instr_mem[index]->print_insn(fp); + inst_size = m_instr_mem[index]->isize; + } else + fprintf(fp, "", pc); + } + pclose(p); + return inst_size; +} + +std::string function_info::get_insn_str(unsigned pc) const { + unsigned index = pc - m_start_PC; + if (index >= m_instr_mem_size) { + char buff[STR_SIZE]; + buff[STR_SIZE - 1] = '\0'; + snprintf(buff, STR_SIZE, "", + m_start_PC + m_instr_mem_size - 1); + return std::string(buff); + } else { + if (m_instr_mem[index] != NULL) { + return m_instr_mem[index]->to_string(); + } else { char buff[STR_SIZE]; - buff[STR_SIZE-1] = '\0'; - snprintf(buff, STR_SIZE, "", m_start_PC + m_instr_mem_size - 1 ); + buff[STR_SIZE - 1] = '\0'; + snprintf(buff, STR_SIZE, "", pc); return std::string(buff); - } else { - if ( m_instr_mem[index] != NULL ) { - return m_instr_mem[index]->to_string(); - } else { - char buff[STR_SIZE]; - buff[STR_SIZE-1] = '\0'; - snprintf(buff, STR_SIZE, "", pc ); - return std::string(buff); - } - } -} - -void gpgpu_ptx_assemble( std::string kname, void *kinfo ) -{ - function_info *func_info = (function_info *)kinfo; - if((function_info *)kinfo == NULL) { - printf("GPGPU-Sim PTX: Warning - missing function definition \'%s\'\n", kname.c_str()); - return; - } - if( func_info->is_extern() ) { - printf("GPGPU-Sim PTX: skipping assembly for extern declared function \'%s\'\n", func_info->get_name().c_str() ); - return; } - func_info->ptx_assemble(); + } +} + +void gpgpu_ptx_assemble(std::string kname, void *kinfo) { + function_info *func_info = (function_info *)kinfo; + if ((function_info *)kinfo == NULL) { + printf("GPGPU-Sim PTX: Warning - missing function definition \'%s\'\n", + kname.c_str()); + return; + } + if (func_info->is_extern()) { + printf( + "GPGPU-Sim PTX: skipping assembly for extern declared function " + "\'%s\'\n", + func_info->get_name().c_str()); + return; + } + func_info->ptx_assemble(); } diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index f4c5c37..6d82374 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -30,13 +32,13 @@ #include "../abstract_hardware_model.h" +#include #include #include -#include #include #include +#include #include -#include //#include "ptx.tab.h" #include "ptx_sim.h" @@ -46,1537 +48,1518 @@ class gpgpu_context; class type_info_key { -public: - type_info_key() - { - m_is_non_arch_reg = false; - m_init = false; - } - type_info_key( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec, int array_dim ) - { - m_is_non_arch_reg = false; - m_init = true; - m_space_spec = space_spec; - m_scalar_type_spec = scalar_type_spec; - m_vector_spec = vector_spec; - m_alignment_spec = alignment_spec; - m_extern_spec = extern_spec; - m_array_dim = array_dim; - m_is_function = 0; - } - void set_is_func() - { - assert(!m_init); - m_init = true; - m_space_spec = undefined_space; - m_scalar_type_spec = 0; - m_vector_spec = 0; - m_alignment_spec = 0; - m_extern_spec = 0; - m_array_dim = 0; - m_is_function = 1; - } - - void set_array_dim( int array_dim ) { m_array_dim = array_dim; } - int get_array_dim() const { assert(m_init); return m_array_dim; } - void set_is_non_arch_reg() { m_is_non_arch_reg = true; } - - bool is_non_arch_reg() const { return m_is_non_arch_reg; } - bool is_reg() const { return m_space_spec == reg_space;} - bool is_param_kernel() const { return m_space_spec == param_space_kernel;} - bool is_param_local() const { return m_space_spec == param_space_local; } - bool is_param_unclassified() const { return m_space_spec == param_space_unclassified; } - bool is_global() const { return m_space_spec == global_space;} - bool is_local() const { return m_space_spec == local_space;} - bool is_shared() const { return m_space_spec == shared_space;} - bool is_const() const { return m_space_spec.get_type() == const_space;} - bool is_tex() const { return m_space_spec == tex_space;} - bool is_func_addr() const { return m_is_function?true:false; } - int scalar_type() const { return m_scalar_type_spec;} - int get_alignment_spec() const { return m_alignment_spec;} - unsigned type_decode( size_t &size, int &t ) const; - static unsigned type_decode( int type, size_t &size, int &t ); - memory_space_t get_memory_space() const { return m_space_spec; } -private: - bool m_init; - memory_space_t m_space_spec; - int m_scalar_type_spec; - int m_vector_spec; - int m_alignment_spec; - int m_extern_spec; - int m_array_dim; - int m_is_function; - bool m_is_non_arch_reg; - - friend struct type_info_key_compare; + public: + type_info_key() { + m_is_non_arch_reg = false; + m_init = false; + } + type_info_key(memory_space_t space_spec, int scalar_type_spec, + int vector_spec, int alignment_spec, int extern_spec, + int array_dim) { + m_is_non_arch_reg = false; + m_init = true; + m_space_spec = space_spec; + m_scalar_type_spec = scalar_type_spec; + m_vector_spec = vector_spec; + m_alignment_spec = alignment_spec; + m_extern_spec = extern_spec; + m_array_dim = array_dim; + m_is_function = 0; + } + void set_is_func() { + assert(!m_init); + m_init = true; + m_space_spec = undefined_space; + m_scalar_type_spec = 0; + m_vector_spec = 0; + m_alignment_spec = 0; + m_extern_spec = 0; + m_array_dim = 0; + m_is_function = 1; + } + + void set_array_dim(int array_dim) { m_array_dim = array_dim; } + int get_array_dim() const { + assert(m_init); + return m_array_dim; + } + void set_is_non_arch_reg() { m_is_non_arch_reg = true; } + + bool is_non_arch_reg() const { return m_is_non_arch_reg; } + bool is_reg() const { return m_space_spec == reg_space; } + bool is_param_kernel() const { return m_space_spec == param_space_kernel; } + bool is_param_local() const { return m_space_spec == param_space_local; } + bool is_param_unclassified() const { + return m_space_spec == param_space_unclassified; + } + bool is_global() const { return m_space_spec == global_space; } + bool is_local() const { return m_space_spec == local_space; } + bool is_shared() const { return m_space_spec == shared_space; } + bool is_const() const { return m_space_spec.get_type() == const_space; } + bool is_tex() const { return m_space_spec == tex_space; } + bool is_func_addr() const { return m_is_function ? true : false; } + int scalar_type() const { return m_scalar_type_spec; } + int get_alignment_spec() const { return m_alignment_spec; } + unsigned type_decode(size_t &size, int &t) const; + static unsigned type_decode(int type, size_t &size, int &t); + memory_space_t get_memory_space() const { return m_space_spec; } + + private: + bool m_init; + memory_space_t m_space_spec; + int m_scalar_type_spec; + int m_vector_spec; + int m_alignment_spec; + int m_extern_spec; + int m_array_dim; + int m_is_function; + bool m_is_non_arch_reg; + + friend struct type_info_key_compare; }; class symbol_table; struct type_info_key_compare { - bool operator()( const type_info_key &a, const type_info_key &b ) const - { - assert( a.m_init && b.m_init ); - if ( a.m_space_spec < b.m_space_spec ) return true; - if ( a.m_scalar_type_spec < b.m_scalar_type_spec ) return true; - if ( a.m_vector_spec < b.m_vector_spec ) return true; - if ( a.m_alignment_spec < b.m_alignment_spec ) return true; - if ( a.m_extern_spec < b.m_extern_spec ) return true; - if ( a.m_array_dim < b.m_array_dim ) return true; - if ( a.m_is_function < b.m_is_function ) return true; - - return false; - } + bool operator()(const type_info_key &a, const type_info_key &b) const { + assert(a.m_init && b.m_init); + if (a.m_space_spec < b.m_space_spec) return true; + if (a.m_scalar_type_spec < b.m_scalar_type_spec) return true; + if (a.m_vector_spec < b.m_vector_spec) return true; + if (a.m_alignment_spec < b.m_alignment_spec) return true; + if (a.m_extern_spec < b.m_extern_spec) return true; + if (a.m_array_dim < b.m_array_dim) return true; + if (a.m_is_function < b.m_is_function) return true; + + return false; + } }; class type_info { -public: - type_info( symbol_table *scope, type_info_key t ) - { - m_type_info = t; - } - const type_info_key &get_key() const { return m_type_info;} - -private: - symbol_table *m_scope; - type_info_key m_type_info; + public: + type_info(symbol_table *scope, type_info_key t) { m_type_info = t; } + const type_info_key &get_key() const { return m_type_info; } + + private: + symbol_table *m_scope; + type_info_key m_type_info; }; enum operand_type { - reg_t, vector_t, builtin_t, address_t, memory_t, float_op_t, double_op_t, int_t, - unsigned_t, symbolic_t, label_t, v_reg_t, v_float_op_t, v_double_op_t, - v_int_t, v_unsigned_t, undef_t + reg_t, + vector_t, + builtin_t, + address_t, + memory_t, + float_op_t, + double_op_t, + int_t, + unsigned_t, + symbolic_t, + label_t, + v_reg_t, + v_float_op_t, + v_double_op_t, + v_int_t, + v_unsigned_t, + undef_t }; class operand_info; class symbol { -public: - symbol( const char *name, const type_info *type, const char *location, unsigned size, gpgpu_context* ctx ) - { - gpgpu_ctx = ctx; - m_uid = get_uid(); - m_name = name; - m_decl_location = location; - m_type = type; - m_size = size; - m_address_valid = false; - m_is_label = false; - m_is_shared = false; - m_is_const = false; - m_is_global = false; - m_is_local = false; - m_is_param_local = false; - m_is_param_kernel = false; - m_is_tex = false; - m_is_func_addr = false; - m_reg_num_valid = false; - m_function = NULL; - m_reg_num=(unsigned)-1; - m_arch_reg_num=(unsigned)-1; - m_address=(unsigned)-1; - m_initializer.clear(); - if ( type ) m_is_shared = type->get_key().is_shared(); - if ( type ) m_is_const = type->get_key().is_const(); - if ( type ) m_is_global = type->get_key().is_global(); - if ( type ) m_is_local = type->get_key().is_local(); - if ( type ) m_is_param_local = type->get_key().is_param_local(); - if ( type ) m_is_param_kernel = type->get_key().is_param_kernel(); - if ( type ) m_is_tex = type->get_key().is_tex(); - if ( type ) m_is_func_addr = type->get_key().is_func_addr(); - } - unsigned get_size_in_bytes() const - { - return m_size; - } - const std::string &name() const { return m_name;} - const std::string &decl_location() const { return m_decl_location;} - const type_info *type() const { return m_type;} - addr_t get_address() const - { - assert( m_is_label || !m_type->get_key().is_reg() ); // todo : other assertions - assert( m_address_valid ); - return m_address; - } - function_info *get_pc() const - { - return m_function; - } - void set_regno( unsigned regno, unsigned arch_regno ) - { - m_reg_num_valid = true; - m_reg_num = regno; - m_arch_reg_num = arch_regno; - } - - void set_address( addr_t addr ) - { - m_address_valid = true; - m_address = addr; - } - void set_label_address( addr_t addr) - { - m_address_valid = true; - m_address = addr; - m_is_label = true; - } - void set_function( function_info *func ) - { - m_function = func; - m_is_func_addr = true; - } - - bool is_label() const { return m_is_label;} - bool is_shared() const { return m_is_shared;} - bool is_sstarr() const { return m_is_sstarr;} - bool is_const() const { return m_is_const;} - bool is_global() const { return m_is_global;} - bool is_local() const { return m_is_local;} - bool is_param_local() const { return m_is_param_local; } - bool is_param_kernel() const { return m_is_param_kernel; } - bool is_tex() const { return m_is_tex;} - bool is_func_addr() const { return m_is_func_addr; } - bool is_reg() const - { - if ( m_type == NULL ) { - return false; - } - return m_type->get_key().is_reg(); - } - bool is_non_arch_reg() const - { - if ( m_type == NULL ) { - return false; - } - return m_type->get_key().is_non_arch_reg(); - } - - void add_initializer( const std::list &init ); - bool has_initializer() const - { - return m_initializer.size() > 0; - } - std::list get_initializer() const - { - return m_initializer; - } - unsigned reg_num() const - { - assert( m_reg_num_valid ); - return m_reg_num; - } - unsigned arch_reg_num() const - { - assert( m_reg_num_valid ); - return m_arch_reg_num; - } - void print_info(FILE *fp) const; - unsigned uid() const { return m_uid; } - -private: - gpgpu_context* gpgpu_ctx; - unsigned get_uid(); - unsigned m_uid; - const type_info *m_type; - unsigned m_size; // in bytes - std::string m_name; - std::string m_decl_location; - - unsigned m_address; - function_info *m_function; // used for function symbols - - bool m_address_valid; - bool m_is_label; - bool m_is_shared; - bool m_is_sstarr; - bool m_is_const; - bool m_is_global; - bool m_is_local; - bool m_is_param_local; - bool m_is_param_kernel; - bool m_is_tex; - bool m_is_func_addr; - unsigned m_reg_num; - unsigned m_arch_reg_num; - bool m_reg_num_valid; - - std::list m_initializer; + public: + symbol(const char *name, const type_info *type, const char *location, + unsigned size, gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_uid = get_uid(); + m_name = name; + m_decl_location = location; + m_type = type; + m_size = size; + m_address_valid = false; + m_is_label = false; + m_is_shared = false; + m_is_const = false; + m_is_global = false; + m_is_local = false; + m_is_param_local = false; + m_is_param_kernel = false; + m_is_tex = false; + m_is_func_addr = false; + m_reg_num_valid = false; + m_function = NULL; + m_reg_num = (unsigned)-1; + m_arch_reg_num = (unsigned)-1; + m_address = (unsigned)-1; + m_initializer.clear(); + if (type) m_is_shared = type->get_key().is_shared(); + if (type) m_is_const = type->get_key().is_const(); + if (type) m_is_global = type->get_key().is_global(); + if (type) m_is_local = type->get_key().is_local(); + if (type) m_is_param_local = type->get_key().is_param_local(); + if (type) m_is_param_kernel = type->get_key().is_param_kernel(); + if (type) m_is_tex = type->get_key().is_tex(); + if (type) m_is_func_addr = type->get_key().is_func_addr(); + } + unsigned get_size_in_bytes() const { return m_size; } + const std::string &name() const { return m_name; } + const std::string &decl_location() const { return m_decl_location; } + const type_info *type() const { return m_type; } + addr_t get_address() const { + assert(m_is_label || + !m_type->get_key().is_reg()); // todo : other assertions + assert(m_address_valid); + return m_address; + } + function_info *get_pc() const { return m_function; } + void set_regno(unsigned regno, unsigned arch_regno) { + m_reg_num_valid = true; + m_reg_num = regno; + m_arch_reg_num = arch_regno; + } + + void set_address(addr_t addr) { + m_address_valid = true; + m_address = addr; + } + void set_label_address(addr_t addr) { + m_address_valid = true; + m_address = addr; + m_is_label = true; + } + void set_function(function_info *func) { + m_function = func; + m_is_func_addr = true; + } + + bool is_label() const { return m_is_label; } + bool is_shared() const { return m_is_shared; } + bool is_sstarr() const { return m_is_sstarr; } + bool is_const() const { return m_is_const; } + bool is_global() const { return m_is_global; } + bool is_local() const { return m_is_local; } + bool is_param_local() const { return m_is_param_local; } + bool is_param_kernel() const { return m_is_param_kernel; } + bool is_tex() const { return m_is_tex; } + bool is_func_addr() const { return m_is_func_addr; } + bool is_reg() const { + if (m_type == NULL) { + return false; + } + return m_type->get_key().is_reg(); + } + bool is_non_arch_reg() const { + if (m_type == NULL) { + return false; + } + return m_type->get_key().is_non_arch_reg(); + } + + void add_initializer(const std::list &init); + bool has_initializer() const { return m_initializer.size() > 0; } + std::list get_initializer() const { return m_initializer; } + unsigned reg_num() const { + assert(m_reg_num_valid); + return m_reg_num; + } + unsigned arch_reg_num() const { + assert(m_reg_num_valid); + return m_arch_reg_num; + } + void print_info(FILE *fp) const; + unsigned uid() const { return m_uid; } + + private: + gpgpu_context *gpgpu_ctx; + unsigned get_uid(); + unsigned m_uid; + const type_info *m_type; + unsigned m_size; // in bytes + std::string m_name; + std::string m_decl_location; + + unsigned m_address; + function_info *m_function; // used for function symbols + + bool m_address_valid; + bool m_is_label; + bool m_is_shared; + bool m_is_sstarr; + bool m_is_const; + bool m_is_global; + bool m_is_local; + bool m_is_param_local; + bool m_is_param_kernel; + bool m_is_tex; + bool m_is_func_addr; + unsigned m_reg_num; + unsigned m_arch_reg_num; + bool m_reg_num_valid; + + std::list m_initializer; }; class symbol_table { -public: - symbol_table(); - symbol_table( const char *scope_name, unsigned entry_point, symbol_table *parent, gpgpu_context* ctx); - void set_name( const char *name ); - const ptx_version &get_ptx_version() const; - unsigned get_sm_target() const; - void set_ptx_version( float ver, unsigned ext ); - void set_sm_target( const char *target, const char *ext, const char *ext2 ); - symbol* lookup( const char *identifier ); - std::string get_scope_name() const { return m_scope_name; } - symbol *add_variable( const char *identifier, const type_info *type, unsigned size, const char *filename, unsigned line ); - void add_function( function_info *func, const char *filename, unsigned linenumber ); - bool add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **symbol_table ); - function_info *lookup_function(std::string name); - type_info *add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ); - type_info *add_type( function_info *func ); - type_info *get_array_type( type_info *base_type, unsigned array_dim ); - void set_label_address( const symbol *label, unsigned addr ); - unsigned next_reg_num() { return ++m_reg_allocator;} - addr_t get_shared_next() { return m_shared_next;} - addr_t get_sstarr_next() { return m_sstarr_next;} - addr_t get_global_next() { return m_global_next;} - addr_t get_local_next() { return m_local_next;} - addr_t get_tex_next() { return m_tex_next;} - void alloc_shared( unsigned num_bytes ) { m_shared_next += num_bytes;} - void alloc_sstarr( unsigned num_bytes ) { m_sstarr_next += num_bytes;} - void alloc_global( unsigned num_bytes ) { m_global_next += num_bytes;} - void alloc_local( unsigned num_bytes ) { m_local_next += num_bytes;} - void alloc_tex( unsigned num_bytes ) { m_tex_next += num_bytes;} - - typedef std::list::iterator iterator; - - iterator global_iterator_begin() { return m_globals.begin();} - iterator global_iterator_end() { return m_globals.end();} - - iterator const_iterator_begin() { return m_consts.begin();} - iterator const_iterator_end() { return m_consts.end();} - - void dump(); - - //Jin: handle instruction group for cdp - symbol_table* start_inst_group(); - symbol_table* end_inst_group(); - - // backward pointer - class gpgpu_context* gpgpu_ctx; - -private: - unsigned m_reg_allocator; - unsigned m_shared_next; - unsigned m_sstarr_next; - unsigned m_const_next; - unsigned m_global_next; - unsigned m_local_next; - unsigned m_tex_next; - - symbol_table *m_parent; - ptx_version m_ptx_version; - std::string m_scope_name; - std::map m_symbols; //map from name of register to pointers to the registers - std::map m_types; - std::list m_globals; - std::list m_consts; - std::map m_function_info_lookup; - std::map m_function_symtab_lookup; - - //Jin: handle instruction group for cdp - unsigned m_inst_group_id; - std::map m_inst_group_symtab; + public: + symbol_table(); + symbol_table(const char *scope_name, unsigned entry_point, + symbol_table *parent, gpgpu_context *ctx); + void set_name(const char *name); + const ptx_version &get_ptx_version() const; + unsigned get_sm_target() const; + void set_ptx_version(float ver, unsigned ext); + void set_sm_target(const char *target, const char *ext, const char *ext2); + symbol *lookup(const char *identifier); + std::string get_scope_name() const { return m_scope_name; } + symbol *add_variable(const char *identifier, const type_info *type, + unsigned size, const char *filename, unsigned line); + void add_function(function_info *func, const char *filename, + unsigned linenumber); + bool add_function_decl(const char *name, int entry_point, + function_info **func_info, + symbol_table **symbol_table); + function_info *lookup_function(std::string name); + type_info *add_type(memory_space_t space_spec, int scalar_type_spec, + int vector_spec, int alignment_spec, int extern_spec); + type_info *add_type(function_info *func); + type_info *get_array_type(type_info *base_type, unsigned array_dim); + void set_label_address(const symbol *label, unsigned addr); + unsigned next_reg_num() { return ++m_reg_allocator; } + addr_t get_shared_next() { return m_shared_next; } + addr_t get_sstarr_next() { return m_sstarr_next; } + addr_t get_global_next() { return m_global_next; } + addr_t get_local_next() { return m_local_next; } + addr_t get_tex_next() { return m_tex_next; } + void alloc_shared(unsigned num_bytes) { m_shared_next += num_bytes; } + void alloc_sstarr(unsigned num_bytes) { m_sstarr_next += num_bytes; } + void alloc_global(unsigned num_bytes) { m_global_next += num_bytes; } + void alloc_local(unsigned num_bytes) { m_local_next += num_bytes; } + void alloc_tex(unsigned num_bytes) { m_tex_next += num_bytes; } + + typedef std::list::iterator iterator; + + iterator global_iterator_begin() { return m_globals.begin(); } + iterator global_iterator_end() { return m_globals.end(); } + + iterator const_iterator_begin() { return m_consts.begin(); } + iterator const_iterator_end() { return m_consts.end(); } + + void dump(); + + // Jin: handle instruction group for cdp + symbol_table *start_inst_group(); + symbol_table *end_inst_group(); + + // backward pointer + class gpgpu_context *gpgpu_ctx; + + private: + unsigned m_reg_allocator; + unsigned m_shared_next; + unsigned m_sstarr_next; + unsigned m_const_next; + unsigned m_global_next; + unsigned m_local_next; + unsigned m_tex_next; + + symbol_table *m_parent; + ptx_version m_ptx_version; + std::string m_scope_name; + std::map + m_symbols; // map from name of register to pointers to the registers + std::map m_types; + std::list m_globals; + std::list m_consts; + std::map m_function_info_lookup; + std::map m_function_symtab_lookup; + + // Jin: handle instruction group for cdp + unsigned m_inst_group_id; + std::map m_inst_group_symtab; }; class operand_info { -public: - operand_info(gpgpu_context* ctx) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = false; - m_immediate_address=false; - m_addr_offset = 0; - m_value.m_symbolic=NULL; - } - operand_info( const symbol *addr, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - if ( addr->is_label() ) { - m_type = label_t; - } else if ( addr->is_shared() ) { - m_type = symbolic_t; - } else if ( addr->is_const() ) { - m_type = symbolic_t; - } else if ( addr->is_global() ) { - m_type = symbolic_t; - } else if ( addr->is_local() ) { - m_type = symbolic_t; - } else if ( addr->is_param_local() ) { - m_type = symbolic_t; - } else if ( addr->is_param_kernel() ) { - m_type = symbolic_t; - } else if ( addr->is_tex() ) { - m_type = symbolic_t; - } else if ( addr->is_func_addr() ) { - m_type = symbolic_t; - } else if ( !addr->is_reg() ) { - m_type = symbolic_t; - } else { - m_type = reg_t; - } - - m_is_non_arch_reg = addr->is_non_arch_reg(); - m_value.m_symbolic = addr; - m_addr_offset = 0; - m_vector = false; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( const symbol *addr1, const symbol *addr2, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_type = memory_t; - m_value.m_vector_symbolic = new const symbol*[8]; - m_value.m_vector_symbolic[0] = addr1; - m_value.m_vector_symbolic[1] = addr2; - m_value.m_vector_symbolic[2] = NULL; - m_value.m_vector_symbolic[3] = NULL; - m_value.m_vector_symbolic[4] = NULL; - m_value.m_vector_symbolic[5] = NULL; - m_value.m_vector_symbolic[6] = NULL; - m_value.m_vector_symbolic[7] = NULL; - m_addr_offset = 0; - m_vector = false; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( int builtin_id, int dim_mod, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = builtin_t; - m_value.m_int = builtin_id; - m_addr_offset = dim_mod; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( const symbol *addr, int offset, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = address_t; - m_value.m_symbolic = addr; - m_addr_offset = offset; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( unsigned x, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = unsigned_t; - m_value.m_unsigned = x; - m_addr_offset = x; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=true; - } - operand_info( int x, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = int_t; - m_value.m_int = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( float x, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = float_op_t; - m_value.m_float = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( double x, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = double_op_t; - m_value.m_double = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = true; - m_type = vector_t; - m_value.m_vector_symbolic = new const symbol*[8]; - m_value.m_vector_symbolic[0] = s1; - m_value.m_vector_symbolic[1] = s2; - m_value.m_vector_symbolic[2] = s3; - m_value.m_vector_symbolic[3] = s4; - m_value.m_vector_symbolic[4] = NULL; - m_value.m_vector_symbolic[5] = NULL; - m_value.m_vector_symbolic[6] = NULL; - m_value.m_vector_symbolic[7] = NULL; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4 ,const symbol *s5,const symbol *s6,const symbol *s7, const symbol *s8, gpgpu_context* ctx) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = true; - m_type = vector_t; - m_value.m_vector_symbolic = new const symbol*[8]; - m_value.m_vector_symbolic[0] = s1; - m_value.m_vector_symbolic[1] = s2; - m_value.m_vector_symbolic[2] = s3; - m_value.m_vector_symbolic[3] = s4; - m_value.m_vector_symbolic[4] = s5; - m_value.m_vector_symbolic[5] = s6; - m_value.m_vector_symbolic[6] = s7; - m_value.m_vector_symbolic[7] = s8; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - - void init(gpgpu_context* ctx) - { - gpgpu_ctx = ctx; - m_uid=(unsigned)-1; - m_valid=false; - m_vector=false; - m_type=undef_t; - m_immediate_address=false; - m_addr_space=undefined_space; - m_operand_lohi=0; - m_double_operand_type=0; - m_operand_neg=false; - m_const_mem_offset=(unsigned)-1; - m_value.m_int=0; - m_value.m_unsigned=(unsigned)-1; - m_value.m_float=0; - m_value.m_double=0; - for(unsigned i=0; i<4; i++){ - m_value.m_vint[i]=0; - m_value.m_vunsigned[i]=0; - m_value.m_vfloat[i]=0; - m_value.m_vdouble[i]=0; - } - m_value.m_symbolic=NULL; - m_value.m_vector_symbolic=NULL; - m_addr_offset=0; - m_neg_pred=0; - m_is_return_var=0; - m_is_non_arch_reg=0; - - } - void make_memory_operand() { m_type = memory_t;} - void set_return() { m_is_return_var = true; } - void set_immediate_addr() {m_immediate_address=true;} - const std::string &name() const - { - assert( m_type == symbolic_t || m_type == reg_t || m_type == address_t || m_type == memory_t || m_type == label_t); - return m_value.m_symbolic->name(); - } - - unsigned get_vect_nelem() const - { - assert( is_vector() ); - if( !m_value.m_vector_symbolic[0] ) return 0; - if( !m_value.m_vector_symbolic[1] ) return 1; - if( !m_value.m_vector_symbolic[2] ) return 2; - if( !m_value.m_vector_symbolic[3] ) return 3; - if( !m_value.m_vector_symbolic[4] ) return 4; - if( !m_value.m_vector_symbolic[5] ) return 5; - if( !m_value.m_vector_symbolic[6] ) return 6; - if( !m_value.m_vector_symbolic[7] ) return 7; - return 8; - } - - const symbol* vec_symbol(int idx) const - { - assert(idx < 8); - const symbol *result = m_value.m_vector_symbolic[idx]; - assert( result != NULL ); - return result; - } - - const std::string &vec_name1() const - { - assert( m_type == vector_t); - return m_value.m_vector_symbolic[0]->name(); - } - - const std::string &vec_name2() const - { - assert( m_type == vector_t); - return m_value.m_vector_symbolic[1]->name(); - } - - const std::string &vec_name3() const - { - assert( m_type == vector_t); - return m_value.m_vector_symbolic[2]->name(); - } - - const std::string &vec_name4() const - { - assert( m_type == vector_t); - return m_value.m_vector_symbolic[3]->name(); - } - - bool is_reg() const - { - if ( m_type == reg_t ) { - return true; - } - if ( m_type != symbolic_t ) { - return false; - } - return m_value.m_symbolic->type()->get_key().is_reg(); - } - bool is_param_local() const - { - if ( m_type != symbolic_t ) - return false; - return m_value.m_symbolic->type()->get_key().is_param_local(); - } - - bool is_param_kernel() const - { - if ( m_type != symbolic_t ) - return false; - return m_value.m_symbolic->type()->get_key().is_param_kernel(); - } - - bool is_vector() const - { - if ( m_vector) return true; + public: + operand_info(gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = false; + m_immediate_address = false; + m_addr_offset = 0; + m_value.m_symbolic = NULL; + } + operand_info(const symbol *addr, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + if (addr->is_label()) { + m_type = label_t; + } else if (addr->is_shared()) { + m_type = symbolic_t; + } else if (addr->is_const()) { + m_type = symbolic_t; + } else if (addr->is_global()) { + m_type = symbolic_t; + } else if (addr->is_local()) { + m_type = symbolic_t; + } else if (addr->is_param_local()) { + m_type = symbolic_t; + } else if (addr->is_param_kernel()) { + m_type = symbolic_t; + } else if (addr->is_tex()) { + m_type = symbolic_t; + } else if (addr->is_func_addr()) { + m_type = symbolic_t; + } else if (!addr->is_reg()) { + m_type = symbolic_t; + } else { + m_type = reg_t; + } + + m_is_non_arch_reg = addr->is_non_arch_reg(); + m_value.m_symbolic = addr; + m_addr_offset = 0; + m_vector = false; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(const symbol *addr1, const symbol *addr2, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_type = memory_t; + m_value.m_vector_symbolic = new const symbol *[8]; + m_value.m_vector_symbolic[0] = addr1; + m_value.m_vector_symbolic[1] = addr2; + m_value.m_vector_symbolic[2] = NULL; + m_value.m_vector_symbolic[3] = NULL; + m_value.m_vector_symbolic[4] = NULL; + m_value.m_vector_symbolic[5] = NULL; + m_value.m_vector_symbolic[6] = NULL; + m_value.m_vector_symbolic[7] = NULL; + m_addr_offset = 0; + m_vector = false; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(int builtin_id, int dim_mod, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = builtin_t; + m_value.m_int = builtin_id; + m_addr_offset = dim_mod; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(const symbol *addr, int offset, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = address_t; + m_value.m_symbolic = addr; + m_addr_offset = offset; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(unsigned x, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = unsigned_t; + m_value.m_unsigned = x; + m_addr_offset = x; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = true; + } + operand_info(int x, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = int_t; + m_value.m_int = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(float x, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = float_op_t; + m_value.m_float = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(double x, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = double_op_t; + m_value.m_double = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(const symbol *s1, const symbol *s2, const symbol *s3, + const symbol *s4, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = true; + m_type = vector_t; + m_value.m_vector_symbolic = new const symbol *[8]; + m_value.m_vector_symbolic[0] = s1; + m_value.m_vector_symbolic[1] = s2; + m_value.m_vector_symbolic[2] = s3; + m_value.m_vector_symbolic[3] = s4; + m_value.m_vector_symbolic[4] = NULL; + m_value.m_vector_symbolic[5] = NULL; + m_value.m_vector_symbolic[6] = NULL; + m_value.m_vector_symbolic[7] = NULL; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(const symbol *s1, const symbol *s2, const symbol *s3, + const symbol *s4, const symbol *s5, const symbol *s6, + const symbol *s7, const symbol *s8, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = true; + m_type = vector_t; + m_value.m_vector_symbolic = new const symbol *[8]; + m_value.m_vector_symbolic[0] = s1; + m_value.m_vector_symbolic[1] = s2; + m_value.m_vector_symbolic[2] = s3; + m_value.m_vector_symbolic[3] = s4; + m_value.m_vector_symbolic[4] = s5; + m_value.m_vector_symbolic[5] = s6; + m_value.m_vector_symbolic[6] = s7; + m_value.m_vector_symbolic[7] = s8; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + + void init(gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_uid = (unsigned)-1; + m_valid = false; + m_vector = false; + m_type = undef_t; + m_immediate_address = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = (unsigned)-1; + m_value.m_int = 0; + m_value.m_unsigned = (unsigned)-1; + m_value.m_float = 0; + m_value.m_double = 0; + for (unsigned i = 0; i < 4; i++) { + m_value.m_vint[i] = 0; + m_value.m_vunsigned[i] = 0; + m_value.m_vfloat[i] = 0; + m_value.m_vdouble[i] = 0; + } + m_value.m_symbolic = NULL; + m_value.m_vector_symbolic = NULL; + m_addr_offset = 0; + m_neg_pred = 0; + m_is_return_var = 0; + m_is_non_arch_reg = 0; + } + void make_memory_operand() { m_type = memory_t; } + void set_return() { m_is_return_var = true; } + void set_immediate_addr() { m_immediate_address = true; } + const std::string &name() const { + assert(m_type == symbolic_t || m_type == reg_t || m_type == address_t || + m_type == memory_t || m_type == label_t); + return m_value.m_symbolic->name(); + } + + unsigned get_vect_nelem() const { + assert(is_vector()); + if (!m_value.m_vector_symbolic[0]) return 0; + if (!m_value.m_vector_symbolic[1]) return 1; + if (!m_value.m_vector_symbolic[2]) return 2; + if (!m_value.m_vector_symbolic[3]) return 3; + if (!m_value.m_vector_symbolic[4]) return 4; + if (!m_value.m_vector_symbolic[5]) return 5; + if (!m_value.m_vector_symbolic[6]) return 6; + if (!m_value.m_vector_symbolic[7]) return 7; + return 8; + } + + const symbol *vec_symbol(int idx) const { + assert(idx < 8); + const symbol *result = m_value.m_vector_symbolic[idx]; + assert(result != NULL); + return result; + } + + const std::string &vec_name1() const { + assert(m_type == vector_t); + return m_value.m_vector_symbolic[0]->name(); + } + + const std::string &vec_name2() const { + assert(m_type == vector_t); + return m_value.m_vector_symbolic[1]->name(); + } + + const std::string &vec_name3() const { + assert(m_type == vector_t); + return m_value.m_vector_symbolic[2]->name(); + } + + const std::string &vec_name4() const { + assert(m_type == vector_t); + return m_value.m_vector_symbolic[3]->name(); + } + + bool is_reg() const { + if (m_type == reg_t) { + return true; + } + if (m_type != symbolic_t) { return false; - } - int reg_num() const { return m_value.m_symbolic->reg_num();} - int reg1_num() const { return m_value.m_vector_symbolic[0]->reg_num();} - int reg2_num() const { return m_value.m_vector_symbolic[1]->reg_num();} - int reg3_num() const { return m_value.m_vector_symbolic[2]?m_value.m_vector_symbolic[2]->reg_num():0; } - int reg4_num() const { return m_value.m_vector_symbolic[3]?m_value.m_vector_symbolic[3]->reg_num():0; } - int reg5_num() const { return m_value.m_vector_symbolic[4]?m_value.m_vector_symbolic[4]->reg_num():0; } - int reg6_num() const { return m_value.m_vector_symbolic[5]?m_value.m_vector_symbolic[5]->reg_num():0; } - int reg7_num() const { return m_value.m_vector_symbolic[6]?m_value.m_vector_symbolic[6]->reg_num():0; } - int reg8_num() const { return m_value.m_vector_symbolic[7]?m_value.m_vector_symbolic[7]->reg_num():0; } - int arch_reg_num() const { return m_value.m_symbolic->arch_reg_num(); } - int arch_reg_num(unsigned n) const { return (m_value.m_vector_symbolic[n])? m_value.m_vector_symbolic[n]->arch_reg_num() : -1; } - bool is_label() const { return m_type == label_t;} - bool is_builtin() const { return m_type == builtin_t;} - - // Memory operand used in ld / st instructions (ex. [__var1]) - bool is_memory_operand() const { return m_type == memory_t;} - - // Memory operand with immediate access (ex. s[0x0004] or g[$r1+=0x0004]) - // This is used by the PTXPlus extension. The operand is assigned an address space during parsing. - bool is_memory_operand2() const { - return (m_addr_space!=undefined_space); - } - - bool is_immediate_address() const { - return m_immediate_address; - } - - bool is_literal() const { return m_type == int_t || - m_type == float_op_t || - m_type == double_op_t || - m_type == unsigned_t;} - bool is_shared() const { - if ( !(m_type == symbolic_t || m_type == address_t || m_type == memory_t) ) { - return false; - } - return m_value.m_symbolic->is_shared(); - } - bool is_sstarr() const { return m_value.m_symbolic->is_sstarr();} - bool is_const() const { return m_value.m_symbolic->is_const();} - bool is_global() const { return m_value.m_symbolic->is_global();} - bool is_local() const { return m_value.m_symbolic->is_local();} - bool is_tex() const { return m_value.m_symbolic->is_tex();} - bool is_return_var() const { return m_is_return_var; } - - bool is_function_address() const - { - if( m_type != symbolic_t ) { - return false; - } - return m_value.m_symbolic->is_func_addr(); - } - - ptx_reg_t get_literal_value() const - { - ptx_reg_t result; - switch ( m_type ) { - case int_t: result.s64 = m_value.m_int; break; - case float_op_t: result.f32 = m_value.m_float; break; - case double_op_t: result.f64 = m_value.m_double; break; - case unsigned_t: result.u32 = m_value.m_unsigned; break; + } + return m_value.m_symbolic->type()->get_key().is_reg(); + } + bool is_param_local() const { + if (m_type != symbolic_t) return false; + return m_value.m_symbolic->type()->get_key().is_param_local(); + } + + bool is_param_kernel() const { + if (m_type != symbolic_t) return false; + return m_value.m_symbolic->type()->get_key().is_param_kernel(); + } + + bool is_vector() const { + if (m_vector) return true; + return false; + } + int reg_num() const { return m_value.m_symbolic->reg_num(); } + int reg1_num() const { return m_value.m_vector_symbolic[0]->reg_num(); } + int reg2_num() const { return m_value.m_vector_symbolic[1]->reg_num(); } + int reg3_num() const { + return m_value.m_vector_symbolic[2] + ? m_value.m_vector_symbolic[2]->reg_num() + : 0; + } + int reg4_num() const { + return m_value.m_vector_symbolic[3] + ? m_value.m_vector_symbolic[3]->reg_num() + : 0; + } + int reg5_num() const { + return m_value.m_vector_symbolic[4] + ? m_value.m_vector_symbolic[4]->reg_num() + : 0; + } + int reg6_num() const { + return m_value.m_vector_symbolic[5] + ? m_value.m_vector_symbolic[5]->reg_num() + : 0; + } + int reg7_num() const { + return m_value.m_vector_symbolic[6] + ? m_value.m_vector_symbolic[6]->reg_num() + : 0; + } + int reg8_num() const { + return m_value.m_vector_symbolic[7] + ? m_value.m_vector_symbolic[7]->reg_num() + : 0; + } + int arch_reg_num() const { return m_value.m_symbolic->arch_reg_num(); } + int arch_reg_num(unsigned n) const { + return (m_value.m_vector_symbolic[n]) + ? m_value.m_vector_symbolic[n]->arch_reg_num() + : -1; + } + bool is_label() const { return m_type == label_t; } + bool is_builtin() const { return m_type == builtin_t; } + + // Memory operand used in ld / st instructions (ex. [__var1]) + bool is_memory_operand() const { return m_type == memory_t; } + + // Memory operand with immediate access (ex. s[0x0004] or g[$r1+=0x0004]) + // This is used by the PTXPlus extension. The operand is assigned an address + // space during parsing. + bool is_memory_operand2() const { return (m_addr_space != undefined_space); } + + bool is_immediate_address() const { return m_immediate_address; } + + bool is_literal() const { + return m_type == int_t || m_type == float_op_t || m_type == double_op_t || + m_type == unsigned_t; + } + bool is_shared() const { + if (!(m_type == symbolic_t || m_type == address_t || m_type == memory_t)) { + return false; + } + return m_value.m_symbolic->is_shared(); + } + bool is_sstarr() const { return m_value.m_symbolic->is_sstarr(); } + bool is_const() const { return m_value.m_symbolic->is_const(); } + bool is_global() const { return m_value.m_symbolic->is_global(); } + bool is_local() const { return m_value.m_symbolic->is_local(); } + bool is_tex() const { return m_value.m_symbolic->is_tex(); } + bool is_return_var() const { return m_is_return_var; } + + bool is_function_address() const { + if (m_type != symbolic_t) { + return false; + } + return m_value.m_symbolic->is_func_addr(); + } + + ptx_reg_t get_literal_value() const { + ptx_reg_t result; + switch (m_type) { + case int_t: + result.s64 = m_value.m_int; + break; + case float_op_t: + result.f32 = m_value.m_float; + break; + case double_op_t: + result.f64 = m_value.m_double; + break; + case unsigned_t: + result.u32 = m_value.m_unsigned; + break; default: - assert(0); - break; - } - return result; - } - int get_int() const { return m_value.m_int;} - int get_addr_offset() const { return m_addr_offset;} - const symbol *get_symbol() const { return m_value.m_symbolic;} - void set_type( enum operand_type type ) - { - m_type = type; - } - enum operand_type get_type() const { - return m_type; - } - void set_neg_pred() - { - assert( m_valid ); - m_neg_pred = true; - } - bool is_neg_pred() const { return m_neg_pred; } - bool is_valid() const { return m_valid; } - - void set_addr_space(enum _memory_space_t set_value) { m_addr_space = set_value; } - enum _memory_space_t get_addr_space() const { return m_addr_space; } - void set_operand_lohi(int set_value) { m_operand_lohi = set_value; } - int get_operand_lohi() const { return m_operand_lohi; } - void set_double_operand_type(int set_value) { m_double_operand_type = set_value; } - int get_double_operand_type() const { return m_double_operand_type; } - void set_operand_neg() { m_operand_neg = true; } - bool get_operand_neg() const { return m_operand_neg; } - void set_const_mem_offset(addr_t set_value) { m_const_mem_offset = set_value; } - addr_t get_const_mem_offset() const { return m_const_mem_offset; } - bool is_non_arch_reg() const { return m_is_non_arch_reg; } - -private: - gpgpu_context* gpgpu_ctx; - unsigned m_uid; - bool m_valid; - bool m_vector; - enum operand_type m_type; - bool m_immediate_address; - enum _memory_space_t m_addr_space; - int m_operand_lohi; - int m_double_operand_type; - bool m_operand_neg; - addr_t m_const_mem_offset; - union { - int m_int; - unsigned int m_unsigned; - float m_float; - double m_double; - int m_vint[4]; - unsigned int m_vunsigned[4]; - float m_vfloat[4]; - double m_vdouble[4]; - const symbol* m_symbolic; - const symbol** m_vector_symbolic; - } m_value; - - int m_addr_offset; - - bool m_neg_pred; - bool m_is_return_var; - bool m_is_non_arch_reg; - - unsigned get_uid(); + assert(0); + break; + } + return result; + } + int get_int() const { return m_value.m_int; } + int get_addr_offset() const { return m_addr_offset; } + const symbol *get_symbol() const { return m_value.m_symbolic; } + void set_type(enum operand_type type) { m_type = type; } + enum operand_type get_type() const { return m_type; } + void set_neg_pred() { + assert(m_valid); + m_neg_pred = true; + } + bool is_neg_pred() const { return m_neg_pred; } + bool is_valid() const { return m_valid; } + + void set_addr_space(enum _memory_space_t set_value) { + m_addr_space = set_value; + } + enum _memory_space_t get_addr_space() const { return m_addr_space; } + void set_operand_lohi(int set_value) { m_operand_lohi = set_value; } + int get_operand_lohi() const { return m_operand_lohi; } + void set_double_operand_type(int set_value) { + m_double_operand_type = set_value; + } + int get_double_operand_type() const { return m_double_operand_type; } + void set_operand_neg() { m_operand_neg = true; } + bool get_operand_neg() const { return m_operand_neg; } + void set_const_mem_offset(addr_t set_value) { + m_const_mem_offset = set_value; + } + addr_t get_const_mem_offset() const { return m_const_mem_offset; } + bool is_non_arch_reg() const { return m_is_non_arch_reg; } + + private: + gpgpu_context *gpgpu_ctx; + unsigned m_uid; + bool m_valid; + bool m_vector; + enum operand_type m_type; + bool m_immediate_address; + enum _memory_space_t m_addr_space; + int m_operand_lohi; + int m_double_operand_type; + bool m_operand_neg; + addr_t m_const_mem_offset; + union { + int m_int; + unsigned int m_unsigned; + float m_float; + double m_double; + int m_vint[4]; + unsigned int m_vunsigned[4]; + float m_vfloat[4]; + double m_vdouble[4]; + const symbol *m_symbolic; + const symbol **m_vector_symbolic; + } m_value; + + int m_addr_offset; + + bool m_neg_pred; + bool m_is_return_var; + bool m_is_non_arch_reg; + + unsigned get_uid(); }; extern const char *g_opcode_string[]; struct basic_block_t { - basic_block_t( unsigned ID, ptx_instruction *begin, ptx_instruction *end, bool entry, bool ex) - { - bb_id = ID; - ptx_begin = begin; - ptx_end = end; - is_entry=entry; - is_exit=ex; - immediatepostdominator_id = -1; - immediatedominator_id = -1; - } - - ptx_instruction* ptx_begin; - ptx_instruction* ptx_end; - std::set predecessor_ids; //indices of other basic blocks in m_basic_blocks array - std::set successor_ids; - std::set postdominator_ids; - std::set dominator_ids; - std::set Tmp_ids; - int immediatepostdominator_id; - int immediatedominator_id; - bool is_entry; - bool is_exit; - unsigned bb_id; - - // if this basic block dom B - bool dom(const basic_block_t *B) { - return (B->dominator_ids.find(this->bb_id) != B->dominator_ids.end()); - } - - // if this basic block pdom B - bool pdom(const basic_block_t *B) { - return (B->postdominator_ids.find(this->bb_id) != B->postdominator_ids.end()); - } + basic_block_t(unsigned ID, ptx_instruction *begin, ptx_instruction *end, + bool entry, bool ex) { + bb_id = ID; + ptx_begin = begin; + ptx_end = end; + is_entry = entry; + is_exit = ex; + immediatepostdominator_id = -1; + immediatedominator_id = -1; + } + + ptx_instruction *ptx_begin; + ptx_instruction *ptx_end; + std::set + predecessor_ids; // indices of other basic blocks in m_basic_blocks array + std::set successor_ids; + std::set postdominator_ids; + std::set dominator_ids; + std::set Tmp_ids; + int immediatepostdominator_id; + int immediatedominator_id; + bool is_entry; + bool is_exit; + unsigned bb_id; + + // if this basic block dom B + bool dom(const basic_block_t *B) { + return (B->dominator_ids.find(this->bb_id) != B->dominator_ids.end()); + } + + // if this basic block pdom B + bool pdom(const basic_block_t *B) { + return (B->postdominator_ids.find(this->bb_id) != + B->postdominator_ids.end()); + } }; struct gpgpu_recon_t { - address_type source_pc; - address_type target_pc; - class ptx_instruction* source_inst; - class ptx_instruction* target_inst; + address_type source_pc; + address_type target_pc; + class ptx_instruction *source_inst; + class ptx_instruction *target_inst; }; class ptx_instruction : public warp_inst_t { -public: - ptx_instruction( int opcode, - const symbol *pred, - int neg_pred, - int pred_mod, - symbol *label, - const std::list &operands, - const operand_info &return_var, - const std::list &options, - const std::list &wmma_options, - const std::list &scalar_type, - memory_space_t space_spec, - const char *file, - unsigned line, - const char *source, - const core_config *config, - gpgpu_context* ctx); - - void print_insn() const; - virtual void print_insn( FILE *fp ) const; - std::string to_string() const; - unsigned inst_size() const { return m_inst_size; } - unsigned uid() const { return m_uid;} - int get_opcode() const { return m_opcode;} - const char *get_opcode_cstr() const - { - if ( m_opcode != -1 ) { - return g_opcode_string[m_opcode]; - } else { - return "label"; - } - } - const char *source_file() const { return m_source_file.c_str();} - unsigned source_line() const { return m_source_line;} - unsigned get_num_operands() const { return m_operands.size();} - bool has_pred() const { return m_pred != NULL;} - operand_info get_pred() const; - bool get_pred_neg() const { return m_neg_pred;} - int get_pred_mod() const { return m_pred_mod;} - const char *get_source() const { return m_source.c_str();} - - typedef std::vector::const_iterator const_iterator; - - const_iterator op_iter_begin() const - { - return m_operands.begin(); - } - - const_iterator op_iter_end() const - { - return m_operands.end(); - } - - const operand_info &dst() const - { - assert( !m_operands.empty() ); - return m_operands[0]; - } - - const operand_info &func_addr() const - { - assert( !m_operands.empty() ); - if( !m_operands[0].is_return_var() ) { - return m_operands[0]; - } else { - assert( m_operands.size() >= 2 ); - return m_operands[1]; - } - } - - operand_info &dst() - { - assert( !m_operands.empty() ); + public: + ptx_instruction(int opcode, const symbol *pred, int neg_pred, int pred_mod, + symbol *label, const std::list &operands, + const operand_info &return_var, const std::list &options, + const std::list &wmma_options, + const std::list &scalar_type, memory_space_t space_spec, + const char *file, unsigned line, const char *source, + const core_config *config, gpgpu_context *ctx); + + void print_insn() const; + virtual void print_insn(FILE *fp) const; + std::string to_string() const; + unsigned inst_size() const { return m_inst_size; } + unsigned uid() const { return m_uid; } + int get_opcode() const { return m_opcode; } + const char *get_opcode_cstr() const { + if (m_opcode != -1) { + return g_opcode_string[m_opcode]; + } else { + return "label"; + } + } + const char *source_file() const { return m_source_file.c_str(); } + unsigned source_line() const { return m_source_line; } + unsigned get_num_operands() const { return m_operands.size(); } + bool has_pred() const { return m_pred != NULL; } + operand_info get_pred() const; + bool get_pred_neg() const { return m_neg_pred; } + int get_pred_mod() const { return m_pred_mod; } + const char *get_source() const { return m_source.c_str(); } + + typedef std::vector::const_iterator const_iterator; + + const_iterator op_iter_begin() const { return m_operands.begin(); } + + const_iterator op_iter_end() const { return m_operands.end(); } + + const operand_info &dst() const { + assert(!m_operands.empty()); + return m_operands[0]; + } + + const operand_info &func_addr() const { + assert(!m_operands.empty()); + if (!m_operands[0].is_return_var()) { return m_operands[0]; - } - - const operand_info &src1() const - { - assert( m_operands.size() > 1 ); + } else { + assert(m_operands.size() >= 2); return m_operands[1]; - } - - const operand_info &src2() const - { - assert( m_operands.size() > 2 ); - return m_operands[2]; - } - - const operand_info &src3() const - { - assert( m_operands.size() > 3 ); - return m_operands[3]; - } - const operand_info &src4() const - { - assert( m_operands.size() > 4 ); - return m_operands[4]; - } - const operand_info &src5() const - { - assert( m_operands.size() > 5 ); - return m_operands[5]; - } - const operand_info &src6() const - { - assert( m_operands.size() > 6 ); - return m_operands[6]; - } - const operand_info &src7() const - { - assert( m_operands.size() > 7 ); - return m_operands[7]; - } - const operand_info &src8() const - { - assert( m_operands.size() > 8 ); - return m_operands[8]; - } - - const operand_info &operand_lookup( unsigned n ) const - { - assert( n < m_operands.size() ); - return m_operands[n]; - } - bool has_return() const - { - return m_return_var.is_valid(); - } - - memory_space_t get_space() const { return m_space_spec;} - unsigned get_vector() const { return m_vector_spec;} - unsigned get_atomic() const { return m_atomic_spec;} - - int get_wmma_type() const { - return m_wmma_type; - } - int get_wmma_layout(int index) const { - return m_wmma_layout[index];//0->Matrix D,1->Matrix C - } - int get_type() const - { - assert( !m_scalar_type.empty() ); - return m_scalar_type.front(); - } - - int get_type2() const - { - assert( m_scalar_type.size()==2 ); - return m_scalar_type.back(); - } - - void assign_bb(basic_block_t* basic_block) //assign instruction to a basic block - { - m_basic_block = basic_block; - } - basic_block_t* get_bb() { return m_basic_block;} - void set_m_instr_mem_index(unsigned index) { - m_instr_mem_index = index; - } - void set_PC( addr_t PC ) - { - m_PC = PC; - } - addr_t get_PC() const - { - return m_PC; - } - - unsigned get_m_instr_mem_index() { return m_instr_mem_index;} - unsigned get_cmpop() const { return m_compare_op;} - const symbol *get_label() const { return m_label;} - bool is_label() const { if(m_label){ assert(m_opcode==-1);return true;} return false;} - bool is_hi() const { return m_hi;} - bool is_lo() const { return m_lo;} - bool is_wide() const { return m_wide;} - bool is_uni() const { return m_uni;} - bool is_exit() const { return m_exit;} - bool is_abs() const { return m_abs;} - bool is_neg() const { return m_neg;} - bool is_to() const { return m_to_option; } - unsigned cache_option() const { return m_cache_option; } - unsigned rounding_mode() const { return m_rounding_mode;} - unsigned saturation_mode() const { return m_saturation_mode;} - unsigned dimension() const { return m_geom_spec;} - unsigned barrier_op() const {return m_barrier_op;} - unsigned shfl_op() const {return m_shfl_op;} - unsigned prmt_op() const {return m_prmt_op;} - enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot }; - enum vote_mode_t vote_mode() const { return m_vote_mode; } - - int membar_level() const { return m_membar_level; } - - bool has_memory_read() const { - if( m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP|| m_opcode==MMA_LD_OP ) - return true; - // Check PTXPlus operand type below - // Source operands are memory operands - ptx_instruction::const_iterator op=op_iter_begin(); - for ( int n=0; op != op_iter_end(); op++, n++ ) { //process operands - if( n > 0 && op->is_memory_operand2()) // source operands only - return true; - } - return false; - } - bool has_memory_write() const { - if( m_opcode == ST_OP || m_opcode==MMA_ST_OP ) return true; - // Check PTXPlus operand type below - // Destination operand is a memory operand - ptx_instruction::const_iterator op=op_iter_begin(); - for ( int n=0; (op!=op_iter_end() && n<1); op++, n++ ) { //process operands - if( n==0 && op->is_memory_operand2()) // source operands only - return true; - } - return false; - } - - -private: - void set_opcode_and_latency(); - void set_bar_type(); - void set_fp_or_int_archop(); - void set_mul_div_or_other_archop(); - - basic_block_t *m_basic_block; - unsigned m_uid; - addr_t m_PC; - std::string m_source_file; - unsigned m_source_line; - std::string m_source; - - const symbol *m_pred; - bool m_neg_pred; - int m_pred_mod; - int m_opcode; - const symbol *m_label; - std::vector m_operands; - operand_info m_return_var; - - std::list m_options; - std::list m_wmma_options; - bool m_wide; - bool m_hi; - bool m_lo; - bool m_exit; - bool m_abs; - bool m_neg; - bool m_uni; //if branch instruction, this evaluates to true for uniform branches (ie jumps) - bool m_to_option; - unsigned m_cache_option; - int m_wmma_type; - int m_wmma_layout[2]; - int m_wmma_configuration; - unsigned m_rounding_mode; - unsigned m_compare_op; - unsigned m_saturation_mode; - unsigned m_barrier_op; - unsigned m_shfl_op; - unsigned m_prmt_op; - - std::list m_scalar_type; - memory_space_t m_space_spec; - int m_geom_spec; - int m_vector_spec; - int m_atomic_spec; - enum vote_mode_t m_vote_mode; - int m_membar_level; - int m_instr_mem_index; //index into m_instr_mem array - unsigned m_inst_size; // bytes - - virtual void pre_decode(); - friend class function_info; - // backward pointer - class gpgpu_context* gpgpu_ctx; + } + } + + operand_info &dst() { + assert(!m_operands.empty()); + return m_operands[0]; + } + + const operand_info &src1() const { + assert(m_operands.size() > 1); + return m_operands[1]; + } + + const operand_info &src2() const { + assert(m_operands.size() > 2); + return m_operands[2]; + } + + const operand_info &src3() const { + assert(m_operands.size() > 3); + return m_operands[3]; + } + const operand_info &src4() const { + assert(m_operands.size() > 4); + return m_operands[4]; + } + const operand_info &src5() const { + assert(m_operands.size() > 5); + return m_operands[5]; + } + const operand_info &src6() const { + assert(m_operands.size() > 6); + return m_operands[6]; + } + const operand_info &src7() const { + assert(m_operands.size() > 7); + return m_operands[7]; + } + const operand_info &src8() const { + assert(m_operands.size() > 8); + return m_operands[8]; + } + + const operand_info &operand_lookup(unsigned n) const { + assert(n < m_operands.size()); + return m_operands[n]; + } + bool has_return() const { return m_return_var.is_valid(); } + + memory_space_t get_space() const { return m_space_spec; } + unsigned get_vector() const { return m_vector_spec; } + unsigned get_atomic() const { return m_atomic_spec; } + + int get_wmma_type() const { return m_wmma_type; } + int get_wmma_layout(int index) const { + return m_wmma_layout[index]; // 0->Matrix D,1->Matrix C + } + int get_type() const { + assert(!m_scalar_type.empty()); + return m_scalar_type.front(); + } + + int get_type2() const { + assert(m_scalar_type.size() == 2); + return m_scalar_type.back(); + } + + void assign_bb( + basic_block_t *basic_block) // assign instruction to a basic block + { + m_basic_block = basic_block; + } + basic_block_t *get_bb() { return m_basic_block; } + void set_m_instr_mem_index(unsigned index) { m_instr_mem_index = index; } + void set_PC(addr_t PC) { m_PC = PC; } + addr_t get_PC() const { return m_PC; } + + unsigned get_m_instr_mem_index() { return m_instr_mem_index; } + unsigned get_cmpop() const { return m_compare_op; } + const symbol *get_label() const { return m_label; } + bool is_label() const { + if (m_label) { + assert(m_opcode == -1); + return true; + } + return false; + } + bool is_hi() const { return m_hi; } + bool is_lo() const { return m_lo; } + bool is_wide() const { return m_wide; } + bool is_uni() const { return m_uni; } + bool is_exit() const { return m_exit; } + bool is_abs() const { return m_abs; } + bool is_neg() const { return m_neg; } + bool is_to() const { return m_to_option; } + unsigned cache_option() const { return m_cache_option; } + unsigned rounding_mode() const { return m_rounding_mode; } + unsigned saturation_mode() const { return m_saturation_mode; } + unsigned dimension() const { return m_geom_spec; } + unsigned barrier_op() const { return m_barrier_op; } + unsigned shfl_op() const { return m_shfl_op; } + unsigned prmt_op() const { return m_prmt_op; } + enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot }; + enum vote_mode_t vote_mode() const { return m_vote_mode; } + + int membar_level() const { return m_membar_level; } + + bool has_memory_read() const { + if (m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP || + m_opcode == MMA_LD_OP) + return true; + // Check PTXPlus operand type below + // Source operands are memory operands + ptx_instruction::const_iterator op = op_iter_begin(); + for (int n = 0; op != op_iter_end(); op++, n++) { // process operands + if (n > 0 && op->is_memory_operand2()) // source operands only + return true; + } + return false; + } + bool has_memory_write() const { + if (m_opcode == ST_OP || m_opcode == MMA_ST_OP) return true; + // Check PTXPlus operand type below + // Destination operand is a memory operand + ptx_instruction::const_iterator op = op_iter_begin(); + for (int n = 0; (op != op_iter_end() && n < 1); + op++, n++) { // process operands + if (n == 0 && op->is_memory_operand2()) // source operands only + return true; + } + return false; + } + + private: + void set_opcode_and_latency(); + void set_bar_type(); + void set_fp_or_int_archop(); + void set_mul_div_or_other_archop(); + + basic_block_t *m_basic_block; + unsigned m_uid; + addr_t m_PC; + std::string m_source_file; + unsigned m_source_line; + std::string m_source; + + const symbol *m_pred; + bool m_neg_pred; + int m_pred_mod; + int m_opcode; + const symbol *m_label; + std::vector m_operands; + operand_info m_return_var; + + std::list m_options; + std::list m_wmma_options; + bool m_wide; + bool m_hi; + bool m_lo; + bool m_exit; + bool m_abs; + bool m_neg; + bool m_uni; // if branch instruction, this evaluates to true for uniform + // branches (ie jumps) + bool m_to_option; + unsigned m_cache_option; + int m_wmma_type; + int m_wmma_layout[2]; + int m_wmma_configuration; + unsigned m_rounding_mode; + unsigned m_compare_op; + unsigned m_saturation_mode; + unsigned m_barrier_op; + unsigned m_shfl_op; + unsigned m_prmt_op; + + std::list m_scalar_type; + memory_space_t m_space_spec; + int m_geom_spec; + int m_vector_spec; + int m_atomic_spec; + enum vote_mode_t m_vote_mode; + int m_membar_level; + int m_instr_mem_index; // index into m_instr_mem array + unsigned m_inst_size; // bytes + + virtual void pre_decode(); + friend class function_info; + // backward pointer + class gpgpu_context *gpgpu_ctx; }; class param_info { -public: - param_info() { m_valid = false; m_value_set=false; m_size = 0; m_is_ptr = false; } - param_info( std::string name, int type, size_t size, bool is_ptr, memory_space_t ptr_space ) - { - m_valid = true; - m_value_set = false; - m_name = name; - m_type = type; - m_size = size; - m_is_ptr = is_ptr; - m_ptr_space = ptr_space; - } - void add_data( param_t v ) { - assert( (!m_value_set) || (m_value.size == v.size) ); // if this fails concurrent kernel launches might execute incorrectly - m_value_set = true; - m_value = v; - } - void add_offset( unsigned offset ) { m_offset = offset; } - unsigned get_offset() { assert(m_valid); return m_offset; } - std::string get_name() const { assert(m_valid); return m_name; } - int get_type() const { assert(m_valid); return m_type; } - param_t get_value() const { assert(m_value_set); return m_value; } - size_t get_size() const { assert(m_valid); return m_size; } - bool is_ptr_shared() const { assert(m_valid); return (m_is_ptr and m_ptr_space == shared_space); } -private: - bool m_valid; - std::string m_name; - int m_type; - size_t m_size; - bool m_value_set; - param_t m_value; - unsigned m_offset; - bool m_is_ptr; - memory_space_t m_ptr_space; + public: + param_info() { + m_valid = false; + m_value_set = false; + m_size = 0; + m_is_ptr = false; + } + param_info(std::string name, int type, size_t size, bool is_ptr, + memory_space_t ptr_space) { + m_valid = true; + m_value_set = false; + m_name = name; + m_type = type; + m_size = size; + m_is_ptr = is_ptr; + m_ptr_space = ptr_space; + } + void add_data(param_t v) { + assert((!m_value_set) || (m_value.size == v.size)); // if this fails + // concurrent kernel + // launches might + // execute incorrectly + m_value_set = true; + m_value = v; + } + void add_offset(unsigned offset) { m_offset = offset; } + unsigned get_offset() { + assert(m_valid); + return m_offset; + } + std::string get_name() const { + assert(m_valid); + return m_name; + } + int get_type() const { + assert(m_valid); + return m_type; + } + param_t get_value() const { + assert(m_value_set); + return m_value; + } + size_t get_size() const { + assert(m_valid); + return m_size; + } + bool is_ptr_shared() const { + assert(m_valid); + return (m_is_ptr and m_ptr_space == shared_space); + } + + private: + bool m_valid; + std::string m_name; + int m_type; + size_t m_size; + bool m_value_set; + param_t m_value; + unsigned m_offset; + bool m_is_ptr; + memory_space_t m_ptr_space; }; class function_info { -public: - function_info(int entry_point, gpgpu_context* ctx ); - const ptx_version &get_ptx_version() const { return m_symtab->get_ptx_version(); } - unsigned get_sm_target() const { return m_symtab->get_sm_target(); } - bool is_extern() const { return m_extern; } - void set_name(const char *name) - { - m_name = name; - } - void set_symtab(symbol_table *symtab ) - { - m_symtab = symtab; - } - std::string get_name() const - { - return m_name; - } - unsigned print_insn( unsigned pc, FILE * fp ) const; - std::string get_insn_str( unsigned pc ) const; - void add_inst( const std::list &instructions ) - { - m_instructions = instructions; - } - std::list::iterator find_next_real_instruction( std::list::iterator i ); - void create_basic_blocks( ); - - void print_basic_blocks(); - - void print_basic_block_links(); - void print_basic_block_dot(); - - operand_info* find_break_target( ptx_instruction * p_break_insn ); //find the target of a break instruction - void connect_basic_blocks( ); //iterate across m_basic_blocks of function, connecting basic blocks together - bool connect_break_targets(); //connecting break instructions with proper targets - - //iterate across m_basic_blocks of function, - //finding dominator blocks, using algorithm of - //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - void find_dominators( ); - void print_dominators(); - void find_idominators(); - void print_idominators(); - - //iterate across m_basic_blocks of function, - //finding postdominator blocks, using algorithm of - //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - void find_postdominators( ); - void print_postdominators(); - - //iterate across m_basic_blocks of function, - //finding immediate postdominator blocks, using algorithm of - //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - void find_ipostdominators( ); - void print_ipostdominators(); - void do_pdom(); //function to call pdom analysis - - unsigned get_num_reconvergence_pairs(); - - void get_reconvergence_pairs(gpgpu_recon_t* recon_points); - - unsigned get_function_size() { return m_instructions.size();} - - void ptx_assemble(); - - unsigned ptx_get_inst_op( ptx_thread_info *thread ); - void add_param( const char *name, struct param_t value ) - { - m_kernel_params[ name ] = value; - } - void add_param_name_type_size( unsigned index, std::string name, int type, size_t size, bool ptr, memory_space_t space ); - void add_param_data( unsigned argn, struct gpgpu_ptx_sim_arg *args ); - void add_return_var( const symbol *rv ) - { - m_return_var_sym = rv; - } - void add_arg( const symbol *arg ) - { - assert( arg != NULL ); - m_args.push_back(arg); - } - void remove_args() - { - m_args.clear(); - } - unsigned num_args() const - { - return m_args.size(); - } - unsigned get_args_aligned_size(); - - const symbol* get_arg( unsigned n ) const - { - assert( n < m_args.size() ); - return m_args[n]; - } - bool has_return() const - { - return m_return_var_sym != NULL; - } - const symbol *get_return_var() const - { - return m_return_var_sym; - } - const ptx_instruction *get_instruction( unsigned PC ) const - { - unsigned index = PC - m_start_PC; - if( index < m_instr_mem_size ) - return m_instr_mem[index]; - return NULL; - } - addr_t get_start_PC() const - { - return m_start_PC; - } - - void finalize( memory_space *param_mem ); - void param_to_shared( memory_space *shared_mem, symbol_table *symtab ); - void list_param( FILE *fout ) const; - void ptx_jit_config(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) ; - - const struct gpgpu_ptx_sim_info* get_kernel_info () const - { - assert (m_kernel_info.maxthreads == maxnt_id); - return &m_kernel_info; - } - - const void set_kernel_info (const struct gpgpu_ptx_sim_info &info) { - m_kernel_info = info; - m_kernel_info.ptx_version = 10*get_ptx_version().ver(); - m_kernel_info.sm_target = get_ptx_version().target(); - // THIS DEPENDS ON ptxas being called after the PTX is parsed. - m_kernel_info.maxthreads = maxnt_id; - } - symbol_table *get_symtab() - { - return m_symtab; - } - - unsigned local_mem_framesize() const - { - return m_local_mem_framesize; - } - void set_framesize( unsigned sz ) - { - m_local_mem_framesize = sz; - } - bool is_entry_point() const { return m_entry_point; } - bool is_pdom_set() const { return pdom_done; } //return pdom flag - void set_pdom() { pdom_done = true; } //set pdom flag - - void add_config_param( size_t size, unsigned alignment ){ - unsigned offset = 0; - if (m_param_configs.size()>0){ - unsigned offset_nom = m_param_configs.back().first + m_param_configs.back().second; - //ensure offset matches alignment requirements - offset = offset_nom%alignment ? (offset_nom/alignment + 1) * alignment : offset_nom; - } - m_param_configs.push_back(std::pair(size, offset)); - } - - std::pair get_param_config(unsigned param_num) const { return m_param_configs[param_num]; } - - void set_maxnt_id(unsigned maxthreads) { maxnt_id = maxthreads;} - unsigned get_maxnt_id() { return maxnt_id;} - // backward pointer - class gpgpu_context* gpgpu_ctx; - -private: - unsigned maxnt_id; - unsigned m_uid; - unsigned m_local_mem_framesize; - bool m_entry_point; - bool m_extern; - bool m_assembled; - bool pdom_done; //flag to check whether pdom is completed or not - std::string m_name; - ptx_instruction **m_instr_mem; - unsigned m_start_PC; - unsigned m_instr_mem_size; - std::map m_kernel_params; - std::map m_ptx_kernel_param_info; - std::vector< std::pair > m_param_configs; - const symbol *m_return_var_sym; - std::vector m_args; - std::list m_instructions; - std::vector m_basic_blocks; - std::list > m_back_edges; - std::map labels; - unsigned num_reconvergence_pairs; - - //Registers/shmem/etc. used (from ptxas -v), loaded from ___.ptxinfo along with ___.ptx - struct gpgpu_ptx_sim_info m_kernel_info; - - symbol_table *m_symtab; - - //parameter size for device kernels - int m_args_aligned_size; - - addr_t m_n; // offset in m_instr_mem (used in do_pdom) + public: + function_info(int entry_point, gpgpu_context *ctx); + const ptx_version &get_ptx_version() const { + return m_symtab->get_ptx_version(); + } + unsigned get_sm_target() const { return m_symtab->get_sm_target(); } + bool is_extern() const { return m_extern; } + void set_name(const char *name) { m_name = name; } + void set_symtab(symbol_table *symtab) { m_symtab = symtab; } + std::string get_name() const { return m_name; } + unsigned print_insn(unsigned pc, FILE *fp) const; + std::string get_insn_str(unsigned pc) const; + void add_inst(const std::list &instructions) { + m_instructions = instructions; + } + std::list::iterator find_next_real_instruction( + std::list::iterator i); + void create_basic_blocks(); + + void print_basic_blocks(); + + void print_basic_block_links(); + void print_basic_block_dot(); + + operand_info *find_break_target( + ptx_instruction *p_break_insn); // find the target of a break instruction + void connect_basic_blocks(); // iterate across m_basic_blocks of function, + // connecting basic blocks together + bool + connect_break_targets(); // connecting break instructions with proper targets + + // iterate across m_basic_blocks of function, + // finding dominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 + void find_dominators(); + void print_dominators(); + void find_idominators(); + void print_idominators(); + + // iterate across m_basic_blocks of function, + // finding postdominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 + void find_postdominators(); + void print_postdominators(); + + // iterate across m_basic_blocks of function, + // finding immediate postdominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + void find_ipostdominators(); + void print_ipostdominators(); + void do_pdom(); // function to call pdom analysis + + unsigned get_num_reconvergence_pairs(); + + void get_reconvergence_pairs(gpgpu_recon_t *recon_points); + + unsigned get_function_size() { return m_instructions.size(); } + + void ptx_assemble(); + + unsigned ptx_get_inst_op(ptx_thread_info *thread); + void add_param(const char *name, struct param_t value) { + m_kernel_params[name] = value; + } + void add_param_name_type_size(unsigned index, std::string name, int type, + size_t size, bool ptr, memory_space_t space); + void add_param_data(unsigned argn, struct gpgpu_ptx_sim_arg *args); + void add_return_var(const symbol *rv) { m_return_var_sym = rv; } + void add_arg(const symbol *arg) { + assert(arg != NULL); + m_args.push_back(arg); + } + void remove_args() { m_args.clear(); } + unsigned num_args() const { return m_args.size(); } + unsigned get_args_aligned_size(); + + const symbol *get_arg(unsigned n) const { + assert(n < m_args.size()); + return m_args[n]; + } + bool has_return() const { return m_return_var_sym != NULL; } + const symbol *get_return_var() const { return m_return_var_sym; } + const ptx_instruction *get_instruction(unsigned PC) const { + unsigned index = PC - m_start_PC; + if (index < m_instr_mem_size) return m_instr_mem[index]; + return NULL; + } + addr_t get_start_PC() const { return m_start_PC; } + + void finalize(memory_space *param_mem); + void param_to_shared(memory_space *shared_mem, symbol_table *symtab); + void list_param(FILE *fout) const; + void ptx_jit_config(std::map mallocPtr_Size, + memory_space *param_mem, gpgpu_t *gpu, dim3 gridDim, + dim3 blockDim); + + const struct gpgpu_ptx_sim_info *get_kernel_info() const { + assert(m_kernel_info.maxthreads == maxnt_id); + return &m_kernel_info; + } + + const void set_kernel_info(const struct gpgpu_ptx_sim_info &info) { + m_kernel_info = info; + m_kernel_info.ptx_version = 10 * get_ptx_version().ver(); + m_kernel_info.sm_target = get_ptx_version().target(); + // THIS DEPENDS ON ptxas being called after the PTX is parsed. + m_kernel_info.maxthreads = maxnt_id; + } + symbol_table *get_symtab() { return m_symtab; } + + unsigned local_mem_framesize() const { return m_local_mem_framesize; } + void set_framesize(unsigned sz) { m_local_mem_framesize = sz; } + bool is_entry_point() const { return m_entry_point; } + bool is_pdom_set() const { return pdom_done; } // return pdom flag + void set_pdom() { pdom_done = true; } // set pdom flag + + void add_config_param(size_t size, unsigned alignment) { + unsigned offset = 0; + if (m_param_configs.size() > 0) { + unsigned offset_nom = + m_param_configs.back().first + m_param_configs.back().second; + // ensure offset matches alignment requirements + offset = offset_nom % alignment ? (offset_nom / alignment + 1) * alignment + : offset_nom; + } + m_param_configs.push_back(std::pair(size, offset)); + } + + std::pair get_param_config(unsigned param_num) const { + return m_param_configs[param_num]; + } + + void set_maxnt_id(unsigned maxthreads) { maxnt_id = maxthreads; } + unsigned get_maxnt_id() { return maxnt_id; } + // backward pointer + class gpgpu_context *gpgpu_ctx; + + private: + unsigned maxnt_id; + unsigned m_uid; + unsigned m_local_mem_framesize; + bool m_entry_point; + bool m_extern; + bool m_assembled; + bool pdom_done; // flag to check whether pdom is completed or not + std::string m_name; + ptx_instruction **m_instr_mem; + unsigned m_start_PC; + unsigned m_instr_mem_size; + std::map m_kernel_params; + std::map m_ptx_kernel_param_info; + std::vector > m_param_configs; + const symbol *m_return_var_sym; + std::vector m_args; + std::list m_instructions; + std::vector m_basic_blocks; + std::list > m_back_edges; + std::map labels; + unsigned num_reconvergence_pairs; + + // Registers/shmem/etc. used (from ptxas -v), loaded from ___.ptxinfo along + // with ___.ptx + struct gpgpu_ptx_sim_info m_kernel_info; + + symbol_table *m_symtab; + + // parameter size for device kernels + int m_args_aligned_size; + + addr_t m_n; // offset in m_instr_mem (used in do_pdom) }; class arg_buffer_t { -public: - arg_buffer_t(gpgpu_context* ctx) : m_src_op(ctx) - { - m_is_reg=false; - m_is_param=false; - m_param_value=NULL; - m_reg_value=ptx_reg_t(); - } - arg_buffer_t( const arg_buffer_t &another, gpgpu_context* ctx ) : m_src_op(ctx) - { - make_copy(another); - } - void make_copy( const arg_buffer_t &another ) - { - m_dst = another.m_dst; - m_src_op = another.m_src_op; - m_is_reg = another.m_is_reg; - m_is_param = another.m_is_param; - m_reg_value = another.m_reg_value; - m_param_bytes = another.m_param_bytes; - if( m_is_param ) { - m_param_value = malloc(m_param_bytes); - memcpy(m_param_value,another.m_param_value,m_param_bytes); - } - } - void operator=( const arg_buffer_t &another ) - { - make_copy(another); - } - ~arg_buffer_t() - { - if( m_is_param ) - free(m_param_value); - } - arg_buffer_t( const symbol *dst_sym, const operand_info &src_op, ptx_reg_t source_value ) : m_src_op(src_op) - { - m_dst = dst_sym; - m_reg_value=ptx_reg_t(); - if( dst_sym->is_reg() ) { - m_is_reg = true; - m_is_param = false; - assert( src_op.is_reg() ); - m_reg_value = source_value; - } else { - m_is_param = true; - m_is_reg = false; - m_param_value = calloc(sizeof(ptx_reg_t),1); - //new (m_param_value) ptx_reg_t(source_value); - memcpy(m_param_value,&source_value,sizeof(ptx_reg_t)); - m_param_bytes = sizeof(ptx_reg_t); + public: + arg_buffer_t(gpgpu_context *ctx) : m_src_op(ctx) { + m_is_reg = false; + m_is_param = false; + m_param_value = NULL; + m_reg_value = ptx_reg_t(); + } + arg_buffer_t(const arg_buffer_t &another, gpgpu_context *ctx) + : m_src_op(ctx) { + make_copy(another); + } + void make_copy(const arg_buffer_t &another) { + m_dst = another.m_dst; + m_src_op = another.m_src_op; + m_is_reg = another.m_is_reg; + m_is_param = another.m_is_param; + m_reg_value = another.m_reg_value; + m_param_bytes = another.m_param_bytes; + if (m_is_param) { + m_param_value = malloc(m_param_bytes); + memcpy(m_param_value, another.m_param_value, m_param_bytes); + } + } + void operator=(const arg_buffer_t &another) { make_copy(another); } + ~arg_buffer_t() { + if (m_is_param) free(m_param_value); + } + arg_buffer_t(const symbol *dst_sym, const operand_info &src_op, + ptx_reg_t source_value) + : m_src_op(src_op) { + m_dst = dst_sym; + m_reg_value = ptx_reg_t(); + if (dst_sym->is_reg()) { + m_is_reg = true; + m_is_param = false; + assert(src_op.is_reg()); + m_reg_value = source_value; + } else { + m_is_param = true; + m_is_reg = false; + m_param_value = calloc(sizeof(ptx_reg_t), 1); + // new (m_param_value) ptx_reg_t(source_value); + memcpy(m_param_value, &source_value, sizeof(ptx_reg_t)); + m_param_bytes = sizeof(ptx_reg_t); + } + } + arg_buffer_t(const symbol *dst_sym, const operand_info &src_op, + void *source_param_value_array, unsigned array_size) + : m_src_op(src_op) { + m_dst = dst_sym; + if (dst_sym->is_reg()) { + m_is_reg = true; + m_is_param = false; + assert(src_op.is_param_local()); + assert(dst_sym->get_size_in_bytes() == array_size); + switch (array_size) { + case 1: + m_reg_value.u8 = *(unsigned char *)source_param_value_array; + break; + case 2: + m_reg_value.u16 = *(unsigned short *)source_param_value_array; + break; + case 4: + m_reg_value.u32 = *(unsigned int *)source_param_value_array; + break; + case 8: + m_reg_value.u64 = *(unsigned long long *)source_param_value_array; + break; + default: + printf( + "GPGPU-Sim PTX: ERROR ** source param size does not match known " + "register sizes\n"); + break; } - } - arg_buffer_t( const symbol *dst_sym, const operand_info &src_op, void *source_param_value_array, unsigned array_size ) : m_src_op(src_op) - { - m_dst = dst_sym; - if( dst_sym->is_reg() ) { - m_is_reg = true; - m_is_param = false; - assert( src_op.is_param_local() ); - assert( dst_sym->get_size_in_bytes() == array_size ); - switch( array_size ) { - case 1: m_reg_value.u8 = *(unsigned char*)source_param_value_array; break; - case 2: m_reg_value.u16 = *(unsigned short*)source_param_value_array; break; - case 4: m_reg_value.u32 = *(unsigned int*)source_param_value_array; break; - case 8: m_reg_value.u64 = *(unsigned long long*)source_param_value_array; break; - default: - printf("GPGPU-Sim PTX: ERROR ** source param size does not match known register sizes\n"); - break; - } - } else { - // param - m_is_param = true; - m_is_reg = false; - m_param_value = calloc(array_size,1); - m_param_bytes = array_size; - memcpy(m_param_value,source_param_value_array,array_size); - } - } - - bool is_reg() const { return m_is_reg; } - ptx_reg_t get_reg() const - { - assert(m_is_reg); - return m_reg_value; - } - - const void *get_param_buffer() const - { - assert(m_is_param); - return m_param_value; - } - size_t get_param_buffer_size() const - { - assert(m_is_param); - return m_param_bytes; - } - - const symbol *get_dst() const { return m_dst; } - -private: - // destination of copy - const symbol *m_dst; - - // source operand - operand_info m_src_op; - - // source information - bool m_is_reg; - bool m_is_param; - - // source is register - ptx_reg_t m_reg_value; - - // source is param - void *m_param_value; - unsigned m_param_bytes; + } else { + // param + m_is_param = true; + m_is_reg = false; + m_param_value = calloc(array_size, 1); + m_param_bytes = array_size; + memcpy(m_param_value, source_param_value_array, array_size); + } + } + + bool is_reg() const { return m_is_reg; } + ptx_reg_t get_reg() const { + assert(m_is_reg); + return m_reg_value; + } + + const void *get_param_buffer() const { + assert(m_is_param); + return m_param_value; + } + size_t get_param_buffer_size() const { + assert(m_is_param); + return m_param_bytes; + } + + const symbol *get_dst() const { return m_dst; } + + private: + // destination of copy + const symbol *m_dst; + + // source operand + operand_info m_src_op; + + // source information + bool m_is_reg; + bool m_is_param; + + // source is register + ptx_reg_t m_reg_value; + + // source is param + void *m_param_value; + unsigned m_param_bytes; }; -typedef std::list< arg_buffer_t > arg_buffer_list_t; -arg_buffer_t copy_arg_to_buffer(ptx_thread_info * thread, operand_info actual_param_op, const symbol * formal_param); -void copy_args_into_buffer_list( const ptx_instruction * pI, - ptx_thread_info * thread, - const function_info * target_func, - arg_buffer_list_t &arg_values ); -void copy_buffer_list_into_frame(ptx_thread_info * thread, arg_buffer_list_t &arg_values); -void copy_buffer_to_frame(ptx_thread_info * thread, const arg_buffer_t &a); - +typedef std::list arg_buffer_list_t; +arg_buffer_t copy_arg_to_buffer(ptx_thread_info *thread, + operand_info actual_param_op, + const symbol *formal_param); +void copy_args_into_buffer_list(const ptx_instruction *pI, + ptx_thread_info *thread, + const function_info *target_func, + arg_buffer_list_t &arg_values); +void copy_buffer_list_into_frame(ptx_thread_info *thread, + arg_buffer_list_t &arg_values); +void copy_buffer_to_frame(ptx_thread_info *thread, const arg_buffer_t &a); struct textureInfo { - unsigned int texel_size; //size in bytes, e.g. (channelDesc.x+y+z+w)/8 - unsigned int Tx,Ty; //tiling factor dimensions of layout of texels per 64B cache block - unsigned int Tx_numbits,Ty_numbits; //log2(T) - unsigned int texel_size_numbits; //log2(texel_size) + unsigned int texel_size; // size in bytes, e.g. (channelDesc.x+y+z+w)/8 + unsigned int Tx, + Ty; // tiling factor dimensions of layout of texels per 64B cache block + unsigned int Tx_numbits, Ty_numbits; // log2(T) + unsigned int texel_size_numbits; // log2(texel_size) }; -extern std::map g_sym_name_to_symbol_table; - +extern std::map g_sym_name_to_symbol_table; -void gpgpu_ptx_assemble( std::string kname, void *kinfo ); +void gpgpu_ptx_assemble(std::string kname, void *kinfo); #include "../option_parser.h" -unsigned ptx_kernel_shmem_size( void *kernel_impl ); -unsigned ptx_kernel_nregs( void *kernel_impl ); +unsigned ptx_kernel_shmem_size(void *kernel_impl); +unsigned ptx_kernel_nregs(void *kernel_impl); #endif diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc index 33bcf45..af66314 100644 --- a/src/cuda-sim/ptx_loader.cc +++ b/src/cuda-sim/ptx_loader.cc @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -26,503 +28,556 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "ptx_loader.h" -#include "ptx_ir.h" -#include "cuda-sim.h" -#include "ptx_parser.h" -#include #include +#include #include #include #include "../../libcuda/gpgpu_context.h" +#include "cuda-sim.h" +#include "ptx_ir.h" +#include "ptx_parser.h" /// extern prototypes -extern int ptx_error( yyscan_t yyscanner, const char *s ); -extern int ptx_lex_init(yyscan_t* scanner); -extern void ptx_set_in(FILE * _in_str ,yyscan_t yyscanner ); -extern int ptx_parse(yyscan_t scanner, ptx_recognizer* recognizer); +extern int ptx_error(yyscan_t yyscanner, const char *s); +extern int ptx_lex_init(yyscan_t *scanner); +extern void ptx_set_in(FILE *_in_str, yyscan_t yyscanner); +extern int ptx_parse(yyscan_t scanner, ptx_recognizer *recognizer); extern int ptx_lex_destroy(yyscan_t scanner); -extern int ptx__scan_string(const char*, yyscan_t scanner); +extern int ptx__scan_string(const char *, yyscan_t scanner); -extern std::map get_duplicate(); +extern std::map get_duplicate(); -typedef void * yyscan_t; -extern int ptxinfo_lex_init(yyscan_t* scanner); -extern void ptxinfo_set_in (FILE * _in_str ,yyscan_t yyscanner ); -extern int ptxinfo_parse(yyscan_t scanner, ptxinfo_data* ptxinfo); +typedef void *yyscan_t; +extern int ptxinfo_lex_init(yyscan_t *scanner); +extern void ptxinfo_set_in(FILE *_in_str, yyscan_t yyscanner); +extern int ptxinfo_parse(yyscan_t scanner, ptxinfo_data *ptxinfo); extern int ptxinfo_lex_destroy(yyscan_t scanner); static bool g_save_embedded_ptx; static int g_occupancy_sm_number; -bool ptxinfo_data::keep_intermediate_files() {return g_keep_intermediate_files;} - -void gpgpu_context::ptx_reg_options(option_parser_t opp) -{ - option_parser_register(opp, "-save_embedded_ptx", OPT_BOOL, &g_save_embedded_ptx, - "saves ptx files embedded in binary as .ptx", - "0"); - option_parser_register(opp, "-keep", OPT_BOOL, &(ptxinfo->g_keep_intermediate_files), - "keep intermediate files created by GPGPU-Sim when interfacing with external programs", - "0"); - option_parser_register(opp, "-gpgpu_ptx_save_converted_ptxplus", OPT_BOOL, - &(ptxinfo->m_ptx_save_converted_ptxplus), - "Saved converted ptxplus to a file", - "0"); - option_parser_register(opp, "-gpgpu_occupancy_sm_number", OPT_INT32, &g_occupancy_sm_number, - "The SM number to pass to ptxas when getting register usage for computing GPU occupancy. " - "This parameter is required in the config.", - "0"); +bool ptxinfo_data::keep_intermediate_files() { + return g_keep_intermediate_files; } -void gpgpu_context::print_ptx_file( const char *p, unsigned source_num, const char *filename ) -{ - printf("\nGPGPU-Sim PTX: file _%u.ptx contents:\n\n", source_num ); - char *s = strdup(p); - char *t = s; - unsigned n=1; - while ( *t != '\0' ) { - char *u = t; - while ( (*u != '\n') && (*u != '\0') ) u++; - unsigned last = (*u == '\0'); - *u = '\0'; - const ptx_instruction *pI = ptx_parser->ptx_instruction_lookup(filename,n); - char pc[64]; - if( pI && pI->get_PC() ) - snprintf(pc,64,"%4u", pI->get_PC() ); - else - snprintf(pc,64," "); - printf(" _%u.ptx %4u (pc=%s): %s\n", source_num, n, pc, t ); - if ( last ) break; - t = u+1; - n++; - } - free(s); - fflush(stdout); +void gpgpu_context::ptx_reg_options(option_parser_t opp) { + option_parser_register(opp, "-save_embedded_ptx", OPT_BOOL, + &g_save_embedded_ptx, + "saves ptx files embedded in binary as .ptx", "0"); + option_parser_register(opp, "-keep", OPT_BOOL, + &(ptxinfo->g_keep_intermediate_files), + "keep intermediate files created by GPGPU-Sim when " + "interfacing with external programs", + "0"); + option_parser_register(opp, "-gpgpu_ptx_save_converted_ptxplus", OPT_BOOL, + &(ptxinfo->m_ptx_save_converted_ptxplus), + "Saved converted ptxplus to a file", "0"); + option_parser_register(opp, "-gpgpu_occupancy_sm_number", OPT_INT32, + &g_occupancy_sm_number, + "The SM number to pass to ptxas when getting register " + "usage for computing GPU occupancy. " + "This parameter is required in the config.", + "0"); } -char* ptxinfo_data::gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptxfilename, const std::string elffilename, const std::string sassfilename) -{ - - printf("GPGPU-Sim PTX: converting EMBEDDED .ptx file to ptxplus \n"); - - char fname_ptxplus[1024]; - snprintf(fname_ptxplus,1024,"_ptxplus_XXXXXX"); - int fd4=mkstemp(fname_ptxplus); - close(fd4); - - // Run cuobjdump_to_ptxplus - char commandline[1024]; - int result; - snprintf(commandline, 1024, "$GPGPUSIM_ROOT/build/$GPGPUSIM_CONFIG/cuobjdump_to_ptxplus/cuobjdump_to_ptxplus %s %s %s %s", - ptxfilename.c_str(), - sassfilename.c_str(), - elffilename.c_str(), - fname_ptxplus); - fflush(stdout); - printf("GPGPU-Sim PTX: calling cuobjdump_to_ptxplus\ncommandline: %s\n", commandline); - result = system(commandline); - if(result){fprintf(stderr, "GPGPU-Sim PTX: ERROR ** could not execute %s\n", commandline); exit(1);} - - - // Get ptxplus from file - std::ifstream fileStream(fname_ptxplus, std::ios::in); - std::string text, line; - while(getline(fileStream,line)) { - text += (line + "\n"); - } - fileStream.close(); - - char* ptxplus_str = new char [strlen(text.c_str())+1]; - strcpy(ptxplus_str, text.c_str()); - - if (!m_ptx_save_converted_ptxplus){ - char rm_commandline[1024]; - - snprintf(rm_commandline,1024,"rm -f %s", fname_ptxplus); - - printf("GPGPU-Sim PTX: removing temporary files using \"%s\"\n", rm_commandline); - int rm_result = system(rm_commandline); - if( rm_result != 0 ) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", rm_result); - exit(1); - } - } - printf("GPGPU-Sim PTX: DONE converting EMBEDDED .ptx file to ptxplus \n"); - - return ptxplus_str; +void gpgpu_context::print_ptx_file(const char *p, unsigned source_num, + const char *filename) { + printf("\nGPGPU-Sim PTX: file _%u.ptx contents:\n\n", source_num); + char *s = strdup(p); + char *t = s; + unsigned n = 1; + while (*t != '\0') { + char *u = t; + while ((*u != '\n') && (*u != '\0')) u++; + unsigned last = (*u == '\0'); + *u = '\0'; + const ptx_instruction *pI = ptx_parser->ptx_instruction_lookup(filename, n); + char pc[64]; + if (pI && pI->get_PC()) + snprintf(pc, 64, "%4u", pI->get_PC()); + else + snprintf(pc, 64, " "); + printf(" _%u.ptx %4u (pc=%s): %s\n", source_num, n, pc, t); + if (last) break; + t = u + 1; + n++; + } + free(s); + fflush(stdout); } - -symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_string( const char *p, unsigned source_num ) -{ - char buf[1024]; - snprintf(buf,1024,"_%u.ptx", source_num ); - if( g_save_embedded_ptx ) { - FILE *fp = fopen(buf,"w"); - fprintf(fp,"%s",p); - fclose(fp); +char *ptxinfo_data::gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus( + const std::string ptxfilename, const std::string elffilename, + const std::string sassfilename) { + printf("GPGPU-Sim PTX: converting EMBEDDED .ptx file to ptxplus \n"); + + char fname_ptxplus[1024]; + snprintf(fname_ptxplus, 1024, "_ptxplus_XXXXXX"); + int fd4 = mkstemp(fname_ptxplus); + close(fd4); + + // Run cuobjdump_to_ptxplus + char commandline[1024]; + int result; + snprintf(commandline, 1024, + "$GPGPUSIM_ROOT/build/$GPGPUSIM_CONFIG/cuobjdump_to_ptxplus/" + "cuobjdump_to_ptxplus %s %s %s %s", + ptxfilename.c_str(), sassfilename.c_str(), elffilename.c_str(), + fname_ptxplus); + fflush(stdout); + printf("GPGPU-Sim PTX: calling cuobjdump_to_ptxplus\ncommandline: %s\n", + commandline); + result = system(commandline); + if (result) { + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** could not execute %s\n", + commandline); + exit(1); + } + + // Get ptxplus from file + std::ifstream fileStream(fname_ptxplus, std::ios::in); + std::string text, line; + while (getline(fileStream, line)) { + text += (line + "\n"); + } + fileStream.close(); + + char *ptxplus_str = new char[strlen(text.c_str()) + 1]; + strcpy(ptxplus_str, text.c_str()); + + if (!m_ptx_save_converted_ptxplus) { + char rm_commandline[1024]; + + snprintf(rm_commandline, 1024, "rm -f %s", fname_ptxplus); + + printf("GPGPU-Sim PTX: removing temporary files using \"%s\"\n", + rm_commandline); + int rm_result = system(rm_commandline); + if (rm_result != 0) { + fprintf(stderr, + "GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", + rm_result); + exit(1); } - symbol_table *symtab=init_parser(buf); - ptx_lex_init(&(ptx_parser->scanner)); - ptx__scan_string(p, ptx_parser->scanner); - int errors = ptx_parse (ptx_parser->scanner, ptx_parser); - if ( errors ) { - char fname[1024]; - snprintf(fname,1024,"_ptx_errors_XXXXXX"); - int fd=mkstemp(fname); - close(fd); - printf("GPGPU-Sim PTX: parser error detected, exiting... but first extracting .ptx to \"%s\"\n", fname); - FILE *ptxfile = fopen(fname,"w"); - fprintf(ptxfile,"%s", p ); - fclose(ptxfile); - abort(); - exit(40); - } - ptx_lex_destroy(ptx_parser->scanner); + } + printf("GPGPU-Sim PTX: DONE converting EMBEDDED .ptx file to ptxplus \n"); - if ( g_debug_execution >= 100 ) - print_ptx_file(p,source_num,buf); + return ptxplus_str; +} - printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n",buf); - return symtab; +symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_string( + const char *p, unsigned source_num) { + char buf[1024]; + snprintf(buf, 1024, "_%u.ptx", source_num); + if (g_save_embedded_ptx) { + FILE *fp = fopen(buf, "w"); + fprintf(fp, "%s", p); + fclose(fp); + } + symbol_table *symtab = init_parser(buf); + ptx_lex_init(&(ptx_parser->scanner)); + ptx__scan_string(p, ptx_parser->scanner); + int errors = ptx_parse(ptx_parser->scanner, ptx_parser); + if (errors) { + char fname[1024]; + snprintf(fname, 1024, "_ptx_errors_XXXXXX"); + int fd = mkstemp(fname); + close(fd); + printf( + "GPGPU-Sim PTX: parser error detected, exiting... but first extracting " + ".ptx to \"%s\"\n", + fname); + FILE *ptxfile = fopen(fname, "w"); + fprintf(ptxfile, "%s", p); + fclose(ptxfile); + abort(); + exit(40); + } + ptx_lex_destroy(ptx_parser->scanner); + + if (g_debug_execution >= 100) print_ptx_file(p, source_num, buf); + + printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n", buf); + return symtab; } -symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_filename( const char *filename ) -{ - symbol_table *symtab=init_parser(filename); - printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n",filename); - return symtab; +symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_filename( + const char *filename) { + symbol_table *symtab = init_parser(filename); + printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n", filename); + return symtab; } void fix_duplicate_errors(char fname2[1024]) { - char tempfile[1024] = "_temp_ptx"; - char commandline[1024]; - - // change the name of the ptx file to _temp_ptx - snprintf(commandline,1024,"mv %s %s",fname2,tempfile); - printf("Running: %s\n", commandline); - int result = system(commandline); - if (result != 0) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", fname2, tempfile); - exit(1); - } - - // store all of the ptx into a char array - FILE *ptxsource = fopen(tempfile,"r"); - fseek(ptxsource, 0, SEEK_END); - long filesize = ftell(ptxsource); - rewind(ptxsource); - char *ptxdata = (char*)malloc((filesize+1)*sizeof(char)); - // Fail if we do not read the file - assert(fread(ptxdata, filesize, 1, ptxsource) == 1); - fclose(ptxsource); - - FILE *ptxdest = fopen(fname2,"w"); - std::map duplicate = get_duplicate(); - unsigned offset; - unsigned oldlinenum = 1; - unsigned linenum; - char *startptr = ptxdata; - char *funcptr; - char *tempptr = ptxdata - 1; - char *lineptr = ptxdata - 1; - - // recreate the ptx file without duplications - for ( std::map::iterator iter = duplicate.begin(); - iter != duplicate.end(); - iter++){ - // find the line of the next error - linenum = iter->first; - for (int i = oldlinenum; i < linenum; i++) { - lineptr = strchr(lineptr + 1, '\n'); - } - - // find the end of the current section to be copied over - // then find the start of the next section that will be copied - if (strcmp("function", iter->second) == 0) { - // get location of most recent .func - while (tempptr < lineptr && tempptr != NULL) { - funcptr = tempptr; - tempptr = strstr(funcptr + 1, ".func"); - } - - // get the start of the previous line - offset = 0; - while (*(funcptr - offset) != '\n') offset++; - - fwrite(startptr, sizeof(char), funcptr - offset + 1 - startptr, ptxdest); - - //find next location of startptr - if (*(lineptr + 3) == ';') { - // for function definitions - startptr = lineptr + 5; - } else if (*(lineptr + 3) == '{') { - // for functions enclosed with curly brackets - offset = 5; - unsigned bracket = 1; - while (bracket != 0) { - if (*(lineptr + offset) == '{') bracket++; - else if (*(lineptr + offset) == '}') bracket--; - offset++; - } - startptr = lineptr + offset + 1; - } else { - printf("GPGPU-Sim PTX: ERROR ** Unrecognized function format\n"); - abort(); - } - } else if (strcmp("variable", iter->second) == 0) { - fwrite(startptr, sizeof(char), (int)(lineptr + 1 - startptr), ptxdest); - - //find next location of startptr - offset = 1; - while (*(lineptr + offset) != '\n') offset++; - startptr = lineptr + offset + 1; - } else { - printf("GPGPU-Sim PTX: ERROR ** Unsupported duplicate type: %s\n", iter->second); - } - - oldlinenum = linenum; - } - // copy over the rest of the file - fwrite(startptr, sizeof(char), ptxdata + filesize - startptr, ptxdest); - - // cleanup - free(ptxdata); - fclose(ptxdest); - snprintf(commandline,1024,"rm -f %s",tempfile); - printf("Running: %s\n", commandline); - result = system(commandline); - if (result != 0) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile); - exit(1); - } -} + char tempfile[1024] = "_temp_ptx"; + char commandline[1024]; + + // change the name of the ptx file to _temp_ptx + snprintf(commandline, 1024, "mv %s %s", fname2, tempfile); + printf("Running: %s\n", commandline); + int result = system(commandline); + if (result != 0) { + fprintf(stderr, + "GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", + fname2, tempfile); + exit(1); + } + + // store all of the ptx into a char array + FILE *ptxsource = fopen(tempfile, "r"); + fseek(ptxsource, 0, SEEK_END); + long filesize = ftell(ptxsource); + rewind(ptxsource); + char *ptxdata = (char *)malloc((filesize + 1) * sizeof(char)); + // Fail if we do not read the file + assert(fread(ptxdata, filesize, 1, ptxsource) == 1); + fclose(ptxsource); + + FILE *ptxdest = fopen(fname2, "w"); + std::map duplicate = get_duplicate(); + unsigned offset; + unsigned oldlinenum = 1; + unsigned linenum; + char *startptr = ptxdata; + char *funcptr; + char *tempptr = ptxdata - 1; + char *lineptr = ptxdata - 1; + + // recreate the ptx file without duplications + for (std::map::iterator iter = duplicate.begin(); + iter != duplicate.end(); iter++) { + // find the line of the next error + linenum = iter->first; + for (int i = oldlinenum; i < linenum; i++) { + lineptr = strchr(lineptr + 1, '\n'); + } + + // find the end of the current section to be copied over + // then find the start of the next section that will be copied + if (strcmp("function", iter->second) == 0) { + // get location of most recent .func + while (tempptr < lineptr && tempptr != NULL) { + funcptr = tempptr; + tempptr = strstr(funcptr + 1, ".func"); + } + + // get the start of the previous line + offset = 0; + while (*(funcptr - offset) != '\n') offset++; + + fwrite(startptr, sizeof(char), funcptr - offset + 1 - startptr, ptxdest); + + // find next location of startptr + if (*(lineptr + 3) == ';') { + // for function definitions + startptr = lineptr + 5; + } else if (*(lineptr + 3) == '{') { + // for functions enclosed with curly brackets + offset = 5; + unsigned bracket = 1; + while (bracket != 0) { + if (*(lineptr + offset) == '{') + bracket++; + else if (*(lineptr + offset) == '}') + bracket--; + offset++; + } + startptr = lineptr + offset + 1; + } else { + printf("GPGPU-Sim PTX: ERROR ** Unrecognized function format\n"); + abort(); + } + } else if (strcmp("variable", iter->second) == 0) { + fwrite(startptr, sizeof(char), (int)(lineptr + 1 - startptr), ptxdest); + + // find next location of startptr + offset = 1; + while (*(lineptr + offset) != '\n') offset++; + startptr = lineptr + offset + 1; + } else { + printf("GPGPU-Sim PTX: ERROR ** Unsupported duplicate type: %s\n", + iter->second); + } + oldlinenum = linenum; + } + // copy over the rest of the file + fwrite(startptr, sizeof(char), ptxdata + filesize - startptr, ptxdest); + + // cleanup + free(ptxdata); + fclose(ptxdest); + snprintf(commandline, 1024, "rm -f %s", tempfile); + printf("Running: %s\n", commandline); + result = system(commandline); + if (result != 0) { + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile); + exit(1); + } +} -//we need the application name here too. -char* get_app_binary_name(){ - char exe_path[1025]; - char *self_exe_path; +// we need the application name here too. +char *get_app_binary_name() { + char exe_path[1025]; + char *self_exe_path; #ifdef __APPLE__ - //AMRUTH: get apple device and check the result. - printf("WARNING: not tested for Apple-mac devices \n"); - abort(); + // AMRUTH: get apple device and check the result. + printf("WARNING: not tested for Apple-mac devices \n"); + abort(); #else - std::stringstream exec_link; - exec_link << "/proc/self/exe"; - ssize_t path_length = readlink(exec_link.str().c_str(), exe_path, 1024); - assert(path_length != -1); - exe_path[path_length] = '\0'; - - char *token = strtok(exe_path, "/"); - while(token !=NULL){ - self_exe_path = token; - token = strtok(NULL,"/"); - } + std::stringstream exec_link; + exec_link << "/proc/self/exe"; + ssize_t path_length = readlink(exec_link.str().c_str(), exe_path, 1024); + assert(path_length != -1); + exe_path[path_length] = '\0'; + + char *token = strtok(exe_path, "/"); + while (token != NULL) { + self_exe_path = token; + token = strtok(NULL, "/"); + } #endif - self_exe_path = strtok(self_exe_path, "."); - printf("self exe links to: %s\n", self_exe_path); - return self_exe_path; + self_exe_path = strtok(self_exe_path, "."); + printf("self exe links to: %s\n", self_exe_path); + return self_exe_path; } -void gpgpu_context::gpgpu_ptx_info_load_from_filename( const char *filename, unsigned sm_version) -{ - std::string ptxas_filename(std::string(filename) + "as"); - char buff[1024], extra_flags[1024]; - extra_flags[0]=0; - if(!device_runtime->g_cdp_enabled) - snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version); - else - snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version); - snprintf(buff,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", - extra_flags, filename, ptxas_filename.c_str()); - int result = system(buff); - if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); - } - - FILE *ptxinfo_in; - ptxinfo->g_ptxinfo_filename = strdup(ptxas_filename.c_str()); - ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename,"r"); - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); +void gpgpu_context::gpgpu_ptx_info_load_from_filename(const char *filename, + unsigned sm_version) { + std::string ptxas_filename(std::string(filename) + "as"); + char buff[1024], extra_flags[1024]; + extra_flags[0] = 0; + if (!device_runtime->g_cdp_enabled) + snprintf(extra_flags, 1024, "--gpu-name=sm_%u", sm_version); + else + snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", sm_version); + snprintf( + buff, 1024, + "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, filename, ptxas_filename.c_str()); + int result = system(buff); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } + + FILE *ptxinfo_in; + ptxinfo->g_ptxinfo_filename = strdup(ptxas_filename.c_str()); + ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename, "r"); + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); } -void gpgpu_context::gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num, unsigned sm_version, int no_of_ptx ) -{ - //do ptxas for individual files instead of one big embedded ptx. This prevents the duplicate defs and declarations. - char ptx_file[1000]; - char *name=get_app_binary_name(); - char commandline[4096], fname[1024], fname2[1024], final_tempfile_ptxinfo[1024], tempfile_ptxinfo[1024]; - for (int index=1; index <= no_of_ptx; index++){ - snprintf(ptx_file, 1000, "%s.%d.sm_%u.ptx", name, index, sm_version); - snprintf(fname,1024,"_ptx_XXXXXX"); - int fd=mkstemp(fname); - close(fd); - - printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname); - snprintf(commandline,4096,"cat %s > %s",ptx_file, fname); - if (system(commandline) !=0) { - printf("ERROR: %s command failed\n", commandline); - exit(0); - } - - snprintf(fname2,1024,"_ptx2_XXXXXX"); - fd=mkstemp(fname2); - close(fd); - char commandline2[4096]; - snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2); - printf("Running: %s\n", commandline2); - int result = system(commandline2); - if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); - printf(" Ensure you have write access to simulation directory\n"); - printf(" and have \'cat\' and \'sed\' in your path.\n"); - exit(1); - } - - snprintf(tempfile_ptxinfo,1024,"%sinfo",fname); - char extra_flags[1024]; - extra_flags[0]=0; +void gpgpu_context::gpgpu_ptxinfo_load_from_string(const char *p_for_info, + unsigned source_num, + unsigned sm_version, + int no_of_ptx) { + // do ptxas for individual files instead of one big embedded ptx. This + // prevents the duplicate defs and declarations. + char ptx_file[1000]; + char *name = get_app_binary_name(); + char commandline[4096], fname[1024], fname2[1024], + final_tempfile_ptxinfo[1024], tempfile_ptxinfo[1024]; + for (int index = 1; index <= no_of_ptx; index++) { + snprintf(ptx_file, 1000, "%s.%d.sm_%u.ptx", name, index, sm_version); + snprintf(fname, 1024, "_ptx_XXXXXX"); + int fd = mkstemp(fname); + close(fd); + + printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", + fname); + snprintf(commandline, 4096, "cat %s > %s", ptx_file, fname); + if (system(commandline) != 0) { + printf("ERROR: %s command failed\n", commandline); + exit(0); + } + + snprintf(fname2, 1024, "_ptx2_XXXXXX"); + fd = mkstemp(fname2); + close(fd); + char commandline2[4096]; + snprintf(commandline2, 4096, + "cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, " + "texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 " + "\\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed " + "'s/const\\[.\\]/const\\[0\\]/g' > %s", + fname, fname2); + printf("Running: %s\n", commandline2); + int result = system(commandline2); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); + printf( + " Ensure you have write access to simulation " + "directory\n"); + printf(" and have \'cat\' and \'sed\' in your path.\n"); + exit(1); + } + + snprintf(tempfile_ptxinfo, 1024, "%sinfo", fname); + char extra_flags[1024]; + extra_flags[0] = 0; #if CUDART_VERSION >= 3000 - if ( g_occupancy_sm_number == 0 ) { - fprintf( stderr, "gpgpusim.config must specify the sm version for the GPU that you use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n" - "The register file size is specifically tied to the sm version used to querry ptxas for register usage.\n" - "A register size/SM mismatch may result in occupancy differences." ); - exit(1); + if (g_occupancy_sm_number == 0) { + fprintf( + stderr, + "gpgpusim.config must specify the sm version for the GPU that you " + "use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n" + "The register file size is specifically tied to the sm version used " + "to querry ptxas for register usage.\n" + "A register size/SM mismatch may result in occupancy differences."); + exit(1); } - if(!device_runtime->g_cdp_enabled) - snprintf(extra_flags,1024,"--gpu-name=sm_%u", g_occupancy_sm_number); + if (!device_runtime->g_cdp_enabled) + snprintf(extra_flags, 1024, "--gpu-name=sm_%u", g_occupancy_sm_number); else - snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",g_occupancy_sm_number); + snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", + g_occupancy_sm_number); #endif - snprintf(commandline,1024,"$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + snprintf(commandline, 1024, + "$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file " + "/dev/null 2> %s", extra_flags, fname2, tempfile_ptxinfo); printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); result = system(commandline); - if( result != 0 ) { - // 65280 = duplicate errors - if (result == 65280) { - FILE *ptxinfo_in; - ptxinfo_in = fopen(tempfile_ptxinfo,"r"); - ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); - - fix_duplicate_errors(fname2); - snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", - extra_flags, fname2, tempfile_ptxinfo); - printf("GPGPU-Sim PTX: regenerating ptxinfo using \"%s\"\n", commandline); - result = system(commandline); - } - if (result != 0) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); - } - } + if (result != 0) { + // 65280 = duplicate errors + if (result == 65280) { + FILE *ptxinfo_in; + ptxinfo_in = fopen(tempfile_ptxinfo, "r"); + ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); + + fix_duplicate_errors(fname2); + snprintf(commandline, 1024, + "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file " + "/dev/null 2> %s", + extra_flags, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: regenerating ptxinfo using \"%s\"\n", + commandline); + result = system(commandline); + } + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } } - - //TODO: duplicate code! move it into a function so that it can be reused! - if(no_of_ptx==0) { - //For CDP, we dump everything. So no_of_ptx will be 0. - snprintf(fname,1024,"_ptx_XXXXXX"); - int fd=mkstemp(fname); - close(fd); - - printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname); - FILE *ptxfile = fopen(fname,"w"); - fprintf(ptxfile,"%s", p_for_info); - fclose(ptxfile); - - snprintf(fname2,1024,"_ptx2_XXXXXX"); - fd=mkstemp(fname2); - close(fd); - char commandline2[4096]; - snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2); - printf("Running: %s\n", commandline2); - int result = system(commandline2); - if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); - printf(" Ensure you have write access to simulation directory\n"); - printf(" and have \'cat\' and \'sed\' in your path.\n"); - exit(1); - } - //char tempfile_ptxinfo[1024]; - snprintf(tempfile_ptxinfo,1024,"%sinfo",fname); - char extra_flags[1024]; - extra_flags[0]=0; - - #if CUDART_VERSION >= 3000 - if (sm_version == 0) sm_version = 20; - if(!device_runtime->g_cdp_enabled) - snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version); - else - snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version); - #endif - - snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", - extra_flags, fname2, tempfile_ptxinfo); - printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); - fflush(stdout); - result = system(commandline); - if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); - } + } + + // TODO: duplicate code! move it into a function so that it can be reused! + if (no_of_ptx == 0) { + // For CDP, we dump everything. So no_of_ptx will be 0. + snprintf(fname, 1024, "_ptx_XXXXXX"); + int fd = mkstemp(fname); + close(fd); + + printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", + fname); + FILE *ptxfile = fopen(fname, "w"); + fprintf(ptxfile, "%s", p_for_info); + fclose(ptxfile); + + snprintf(fname2, 1024, "_ptx2_XXXXXX"); + fd = mkstemp(fname2); + close(fd); + char commandline2[4096]; + snprintf(commandline2, 4096, + "cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, " + "texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 " + "\\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed " + "'s/const\\[.\\]/const\\[0\\]/g' > %s", + fname, fname2); + printf("Running: %s\n", commandline2); + int result = system(commandline2); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); + printf( + " Ensure you have write access to simulation " + "directory\n"); + printf(" and have \'cat\' and \'sed\' in your path.\n"); + exit(1); } + // char tempfile_ptxinfo[1024]; + snprintf(tempfile_ptxinfo, 1024, "%sinfo", fname); + char extra_flags[1024]; + extra_flags[0] = 0; - //Now that we got resource usage per kernel in a ptx file, we dump all into one file and pass it to rest of the code as usual. - if(no_of_ptx>0){ - char commandline3[4096]; - snprintf(final_tempfile_ptxinfo,1024,"f_tempfile_ptx"); - snprintf(commandline3,4096, "cat *info > %s", final_tempfile_ptxinfo); - if (system(commandline3)!=0) { - printf("ERROR: Either we dont have info files or cat is not working \n"); - printf("ERROR: %s command failed\n",commandline3); - exit(1); - } - } - - if(no_of_ptx>0) - ptxinfo->g_ptxinfo_filename = final_tempfile_ptxinfo; +#if CUDART_VERSION >= 3000 + if (sm_version == 0) sm_version = 20; + if (!device_runtime->g_cdp_enabled) + snprintf(extra_flags, 1024, "--gpu-name=sm_%u", sm_version); else - ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; - FILE *ptxinfo_in; - ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename,"r"); - - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); - - snprintf(commandline,1024,"rm -f *info"); - if( system(commandline) != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while removing temporary info files\n"); - exit(1); + snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", + sm_version); +#endif + + snprintf( + commandline, 1024, + "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); + fflush(stdout); + result = system(commandline); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); } - if( ! g_save_embedded_ptx ) { - if(no_of_ptx>0) - snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, final_tempfile_ptxinfo); - else - snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, tempfile_ptxinfo); - printf("GPGPU-Sim PTX: removing ptxinfo using \"%s\"\n", commandline); - if( system(commandline) != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while removing temporary files\n"); - exit(1); - } + } + + // Now that we got resource usage per kernel in a ptx file, we dump all into + // one file and pass it to rest of the code as usual. + if (no_of_ptx > 0) { + char commandline3[4096]; + snprintf(final_tempfile_ptxinfo, 1024, "f_tempfile_ptx"); + snprintf(commandline3, 4096, "cat *info > %s", final_tempfile_ptxinfo); + if (system(commandline3) != 0) { + printf("ERROR: Either we dont have info files or cat is not working \n"); + printf("ERROR: %s command failed\n", commandline3); + exit(1); + } + } + + if (no_of_ptx > 0) + ptxinfo->g_ptxinfo_filename = final_tempfile_ptxinfo; + else + ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; + FILE *ptxinfo_in; + ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename, "r"); + + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); + + snprintf(commandline, 1024, "rm -f *info"); + if (system(commandline) != 0) { + printf("GPGPU-Sim PTX: ERROR ** while removing temporary info files\n"); + exit(1); + } + if (!g_save_embedded_ptx) { + if (no_of_ptx > 0) + snprintf(commandline, 1024, "rm -f %s %s %s", fname, fname2, + final_tempfile_ptxinfo); + else + snprintf(commandline, 1024, "rm -f %s %s %s", fname, fname2, + tempfile_ptxinfo); + printf("GPGPU-Sim PTX: removing ptxinfo using \"%s\"\n", commandline); + if (system(commandline) != 0) { + printf("GPGPU-Sim PTX: ERROR ** while removing temporary files\n"); + exit(1); } + } } diff --git a/src/cuda-sim/ptx_loader.h b/src/cuda-sim/ptx_loader.h index d8f1cbc..17c4524 100644 --- a/src/cuda-sim/ptx_loader.h +++ b/src/cuda-sim/ptx_loader.h @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -31,22 +33,22 @@ #define PTXINFO_LINEBUF_SIZE 1024 class gpgpu_context; -typedef void * yyscan_t; -class ptxinfo_data{ - public: - ptxinfo_data(gpgpu_context* ctx) { - gpgpu_ctx = ctx; - } - yyscan_t scanner; - char linebuf[PTXINFO_LINEBUF_SIZE]; - unsigned col; - const char *g_ptxinfo_filename; - class gpgpu_context* gpgpu_ctx; - bool g_keep_intermediate_files; - bool m_ptx_save_converted_ptxplus; - void ptxinfo_addinfo(); - bool keep_intermediate_files(); - char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptx_str, const std::string sass_str, const std::string elf_str); +typedef void* yyscan_t; +class ptxinfo_data { + public: + ptxinfo_data(gpgpu_context* ctx) { gpgpu_ctx = ctx; } + yyscan_t scanner; + char linebuf[PTXINFO_LINEBUF_SIZE]; + unsigned col; + const char* g_ptxinfo_filename; + class gpgpu_context* gpgpu_ctx; + bool g_keep_intermediate_files; + bool m_ptx_save_converted_ptxplus; + void ptxinfo_addinfo(); + bool keep_intermediate_files(); + char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus( + const std::string ptx_str, const std::string sass_str, + const std::string elf_str); }; #endif diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index a4f4a0c..c135a6c 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -26,998 +28,971 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "ptx_parser.h" -#include "ptx_ir.h" #include "../../libcuda/gpgpu_context.h" +#include "ptx_ir.h" -typedef void * yyscan_t; -#include "ptx.tab.h" +typedef void *yyscan_t; #include +#include "ptx.tab.h" -extern int ptx_get_lineno (yyscan_t yyscanner ); -extern YYSTYPE* ptx_get_lval (yyscan_t yyscanner ); -extern int ptx_error( yyscan_t yyscanner, const char *s ); -extern int ptx_lex_init(yyscan_t* scanner); -extern void ptx_set_in(FILE * _in_str ,yyscan_t yyscanner ); -extern FILE *ptx_get_in (yyscan_t yyscanner ); -extern int ptx_parse(yyscan_t scanner, ptx_recognizer* recognizer); +extern int ptx_get_lineno(yyscan_t yyscanner); +extern YYSTYPE *ptx_get_lval(yyscan_t yyscanner); +extern int ptx_error(yyscan_t yyscanner, const char *s); +extern int ptx_lex_init(yyscan_t *scanner); +extern void ptx_set_in(FILE *_in_str, yyscan_t yyscanner); +extern FILE *ptx_get_in(yyscan_t yyscanner); +extern int ptx_parse(yyscan_t scanner, ptx_recognizer *recognizer); extern int ptx_lex_destroy(yyscan_t scanner); -void ptx_recognizer::set_ptx_warp_size(const struct core_config * warp_size) -{ - g_shader_core_config=warp_size; -} - - -#define PTX_PARSE_DPRINTF(...) \ - if( g_debug_ir_generation ) { \ - printf(" %s:%u => ",gpgpu_ctx->g_filename,ptx_get_lineno(scanner)); \ - printf(" (%s:%u) ", __FILE__, __LINE__); \ - printf(__VA_ARGS__); \ - printf("\n"); \ - fflush(stdout); \ - } - -static std::map g_ptx_token_decode; - -const char *decode_token( int type ) -{ - return g_ptx_token_decode[type].c_str(); -} - -void ptx_recognizer::read_parser_environment_variables() -{ - gpgpu_ctx->g_filename = getenv("PTX_SIM_KERNELFILE"); - char *dbg_level = getenv("PTX_SIM_DEBUG"); - if ( dbg_level && strlen(dbg_level) ) { - int debug_execution=0; - sscanf(dbg_level,"%d", &debug_execution); - if ( debug_execution >= 30 ) - g_debug_ir_generation=true; - } -} - -void ptx_recognizer::init_directive_state() -{ - PTX_PARSE_DPRINTF("init_directive_state"); - g_space_spec=undefined_space; - g_ptr_spec=undefined_space; - g_scalar_type_spec=-1; - g_vector_spec=-1; - g_opcode=-1; - g_alignment_spec = -1; - g_size = -1; - g_extern_spec = 0; - g_scalar_type.clear(); - g_operands.clear(); - g_last_symbol = NULL; -} - -void ptx_recognizer::init_instruction_state() -{ - PTX_PARSE_DPRINTF("init_instruction_state"); - g_pred = NULL; - g_neg_pred = 0; - g_pred_mod = -1; - g_label = NULL; - g_opcode = -1; - g_options.clear(); - g_wmma_options.clear(); - g_return_var = operand_info(gpgpu_ctx); - init_directive_state(); -} - -symbol_table * gpgpu_context::init_parser( const char *ptx_filename ) -{ - g_filename = strdup(ptx_filename); - if (g_global_allfiles_symbol_table == NULL) { - g_global_allfiles_symbol_table = new symbol_table("global_allfiles", 0, NULL, this); - ptx_parser->g_global_symbol_table = ptx_parser->g_current_symbol_table = g_global_allfiles_symbol_table; - } - /*else { - g_global_symbol_table = g_current_symbol_table = new symbol_table("global",0,g_global_allfiles_symbol_table); - }*/ - -#define DEF(X,Y) g_ptx_token_decode[X] = Y; +void ptx_recognizer::set_ptx_warp_size(const struct core_config *warp_size) { + g_shader_core_config = warp_size; +} + +#define PTX_PARSE_DPRINTF(...) \ + if (g_debug_ir_generation) { \ + printf(" %s:%u => ", gpgpu_ctx->g_filename, ptx_get_lineno(scanner)); \ + printf(" (%s:%u) ", __FILE__, __LINE__); \ + printf(__VA_ARGS__); \ + printf("\n"); \ + fflush(stdout); \ + } + +static std::map g_ptx_token_decode; + +const char *decode_token(int type) { return g_ptx_token_decode[type].c_str(); } + +void ptx_recognizer::read_parser_environment_variables() { + gpgpu_ctx->g_filename = getenv("PTX_SIM_KERNELFILE"); + char *dbg_level = getenv("PTX_SIM_DEBUG"); + if (dbg_level && strlen(dbg_level)) { + int debug_execution = 0; + sscanf(dbg_level, "%d", &debug_execution); + if (debug_execution >= 30) g_debug_ir_generation = true; + } +} + +void ptx_recognizer::init_directive_state() { + PTX_PARSE_DPRINTF("init_directive_state"); + g_space_spec = undefined_space; + g_ptr_spec = undefined_space; + g_scalar_type_spec = -1; + g_vector_spec = -1; + g_opcode = -1; + g_alignment_spec = -1; + g_size = -1; + g_extern_spec = 0; + g_scalar_type.clear(); + g_operands.clear(); + g_last_symbol = NULL; +} + +void ptx_recognizer::init_instruction_state() { + PTX_PARSE_DPRINTF("init_instruction_state"); + g_pred = NULL; + g_neg_pred = 0; + g_pred_mod = -1; + g_label = NULL; + g_opcode = -1; + g_options.clear(); + g_wmma_options.clear(); + g_return_var = operand_info(gpgpu_ctx); + init_directive_state(); +} + +symbol_table *gpgpu_context::init_parser(const char *ptx_filename) { + g_filename = strdup(ptx_filename); + if (g_global_allfiles_symbol_table == NULL) { + g_global_allfiles_symbol_table = + new symbol_table("global_allfiles", 0, NULL, this); + ptx_parser->g_global_symbol_table = ptx_parser->g_current_symbol_table = + g_global_allfiles_symbol_table; + } +/*else { + g_global_symbol_table = g_current_symbol_table = new +symbol_table("global",0,g_global_allfiles_symbol_table); +}*/ + +#define DEF(X, Y) g_ptx_token_decode[X] = Y; #include "ptx_parser_decode.def" #undef DEF - g_ptx_token_decode[undefined_space] = "undefined_space"; - g_ptx_token_decode[undefined_space] = "undefined_space=0"; - g_ptx_token_decode[reg_space] = "reg_space"; - g_ptx_token_decode[local_space] = "local_space"; - g_ptx_token_decode[shared_space] = "shared_space"; - g_ptx_token_decode[param_space_unclassified] = "param_space_unclassified"; - g_ptx_token_decode[param_space_kernel] = "param_space_kernel"; - g_ptx_token_decode[param_space_local] = "param_space_local"; - g_ptx_token_decode[const_space] = "const_space"; - g_ptx_token_decode[tex_space] = "tex_space"; - g_ptx_token_decode[surf_space] = "surf_space"; - g_ptx_token_decode[global_space] = "global_space"; - g_ptx_token_decode[generic_space] = "generic_space"; - g_ptx_token_decode[instruction_space] = "instruction_space"; - - ptx_lex_init(&(ptx_parser->scanner)); - ptx_parser->init_directive_state(); - ptx_parser->init_instruction_state(); - - FILE *ptx_in; - ptx_in = fopen(ptx_filename, "r"); - ptx_set_in(ptx_in, ptx_parser->scanner); - ptx_parse(ptx_parser->scanner, ptx_parser); - ptx_in = ptx_get_in(ptx_parser->scanner); - ptx_lex_destroy(ptx_parser->scanner); - fclose(ptx_in); - return ptx_parser->g_global_symbol_table; -} - - -void ptx_recognizer::start_function( int entry_point ) -{ - PTX_PARSE_DPRINTF("start_function"); - init_directive_state(); - init_instruction_state(); - g_entry_point = entry_point; - g_func_info = NULL; - g_entry_func_param_index=0; -} - -void ptx_recognizer::add_function_name( const char *name ) -{ - PTX_PARSE_DPRINTF("add_function_name %s %s", name, ((g_entry_point==1)?"(entrypoint)":((g_entry_point==2)?"(extern)":""))); - bool prior_decl = g_global_symbol_table->add_function_decl( name, g_entry_point, &g_func_info, &g_current_symbol_table ); - if( g_add_identifier_cached__identifier ) { - add_identifier( g_add_identifier_cached__identifier, - g_add_identifier_cached__array_dim, - g_add_identifier_cached__array_ident ); - free( g_add_identifier_cached__identifier ); - g_add_identifier_cached__identifier = NULL; - g_func_info->add_return_var( g_last_symbol ); - init_directive_state(); - } - if( prior_decl ) { - g_func_info->remove_args(); - } - g_global_symbol_table->add_function( g_func_info, gpgpu_ctx->g_filename, ptx_get_lineno(scanner) ); -} - -//Jin: handle instruction group for cdp + g_ptx_token_decode[undefined_space] = "undefined_space"; + g_ptx_token_decode[undefined_space] = "undefined_space=0"; + g_ptx_token_decode[reg_space] = "reg_space"; + g_ptx_token_decode[local_space] = "local_space"; + g_ptx_token_decode[shared_space] = "shared_space"; + g_ptx_token_decode[param_space_unclassified] = "param_space_unclassified"; + g_ptx_token_decode[param_space_kernel] = "param_space_kernel"; + g_ptx_token_decode[param_space_local] = "param_space_local"; + g_ptx_token_decode[const_space] = "const_space"; + g_ptx_token_decode[tex_space] = "tex_space"; + g_ptx_token_decode[surf_space] = "surf_space"; + g_ptx_token_decode[global_space] = "global_space"; + g_ptx_token_decode[generic_space] = "generic_space"; + g_ptx_token_decode[instruction_space] = "instruction_space"; + + ptx_lex_init(&(ptx_parser->scanner)); + ptx_parser->init_directive_state(); + ptx_parser->init_instruction_state(); + + FILE *ptx_in; + ptx_in = fopen(ptx_filename, "r"); + ptx_set_in(ptx_in, ptx_parser->scanner); + ptx_parse(ptx_parser->scanner, ptx_parser); + ptx_in = ptx_get_in(ptx_parser->scanner); + ptx_lex_destroy(ptx_parser->scanner); + fclose(ptx_in); + return ptx_parser->g_global_symbol_table; +} + +void ptx_recognizer::start_function(int entry_point) { + PTX_PARSE_DPRINTF("start_function"); + init_directive_state(); + init_instruction_state(); + g_entry_point = entry_point; + g_func_info = NULL; + g_entry_func_param_index = 0; +} + +void ptx_recognizer::add_function_name(const char *name) { + PTX_PARSE_DPRINTF( + "add_function_name %s %s", name, + ((g_entry_point == 1) ? "(entrypoint)" + : ((g_entry_point == 2) ? "(extern)" : ""))); + bool prior_decl = g_global_symbol_table->add_function_decl( + name, g_entry_point, &g_func_info, &g_current_symbol_table); + if (g_add_identifier_cached__identifier) { + add_identifier(g_add_identifier_cached__identifier, + g_add_identifier_cached__array_dim, + g_add_identifier_cached__array_ident); + free(g_add_identifier_cached__identifier); + g_add_identifier_cached__identifier = NULL; + g_func_info->add_return_var(g_last_symbol); + init_directive_state(); + } + if (prior_decl) { + g_func_info->remove_args(); + } + g_global_symbol_table->add_function(g_func_info, gpgpu_ctx->g_filename, + ptx_get_lineno(scanner)); +} + +// Jin: handle instruction group for cdp void ptx_recognizer::start_inst_group() { - PTX_PARSE_DPRINTF("start_instruction_group"); - g_current_symbol_table = g_current_symbol_table->start_inst_group(); + PTX_PARSE_DPRINTF("start_instruction_group"); + g_current_symbol_table = g_current_symbol_table->start_inst_group(); } void ptx_recognizer::end_inst_group() { - PTX_PARSE_DPRINTF("end_instruction_group"); - g_current_symbol_table = g_current_symbol_table->end_inst_group(); -} - -void ptx_recognizer::add_directive() -{ - PTX_PARSE_DPRINTF("add_directive"); - init_directive_state(); -} - -#define mymax(a,b) ((a)>(b)?(a):(b)) - -void ptx_recognizer::end_function() -{ - PTX_PARSE_DPRINTF("end_function"); - - init_directive_state(); - init_instruction_state(); - g_max_regs_per_thread = mymax( g_max_regs_per_thread, (g_current_symbol_table->next_reg_num()-1)); - g_func_info->add_inst( g_instructions ); - g_instructions.clear(); - gpgpu_ptx_assemble( g_func_info->get_name(), g_func_info ); - g_current_symbol_table = g_global_symbol_table; - - PTX_PARSE_DPRINTF("function %s, PC = %d\n", g_func_info->get_name().c_str(), g_func_info->get_start_PC()); -} - -#define parse_error(msg, ...) parse_error_impl(__FILE__,__LINE__, msg, ##__VA_ARGS__) -#define parse_assert(cond,msg, ...) parse_assert_impl((cond),__FILE__,__LINE__, msg, ##__VA_ARGS__) - -void ptx_recognizer::parse_error_impl( const char *file, unsigned line, const char *msg, ... ) -{ - va_list ap; - char buf[1024]; - va_start(ap,msg); - vsnprintf(buf,1024,msg,ap); - va_end(ap); - - g_error_detected = 1; - printf("%s:%u: Parse error: %s (%s:%u)\n\n", gpgpu_ctx->g_filename, ptx_get_lineno(scanner), buf, file, line); - ptx_error(scanner, NULL); - abort(); - exit(1); -} - -void ptx_recognizer::parse_assert_impl( int test_value, const char *file, unsigned line, const char *msg, ... ) -{ - va_list ap; - char buf[1024]; - va_start(ap,msg); - vsnprintf(buf,1024,msg,ap); - va_end(ap); - - if ( test_value == 0 ) - parse_error_impl(file,line, msg); -} - - - -void ptx_recognizer::set_return() -{ - parse_assert( (g_opcode == CALL_OP || g_opcode == CALLP_OP), "only call can have return value"); - g_operands.front().set_return(); - g_return_var = g_operands.front(); -} - - -const ptx_instruction *ptx_recognizer::ptx_instruction_lookup( const char *filename, unsigned linenumber ) -{ - std::map >::iterator f=g_inst_lookup.find(filename); - if( f == g_inst_lookup.end() ) - return NULL; - std::map::iterator l=f->second.find(linenumber); - if( l == f->second.end() ) - return NULL; - return l->second; -} - -void ptx_recognizer::add_instruction() -{ - PTX_PARSE_DPRINTF("add_instruction: %s", ((g_opcode>0)?g_opcode_string[g_opcode]:"